2024-04-26T15:16:03.073 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-26T15:16:03.074 DEBUG:teuthology.run:Teuthology command: teuthology --description orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --name phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi --archive /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674785 --owner scheduled_phlogistonjohn@teuthology --verbose -- /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674785/orig.config.yaml 2024-04-26T15:16:03.112 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T15:16:03.198 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674785 branch: wip-phlogistonjohn-testing-2024-04-25-1915 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: jmulligan@redhat.com first_in_suite: false job_id: '7674785' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-phlogistonjohn-testing-2024-04-25-1915 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: a430f9ed84de3008ca5b632b1ed304bac0e753f6 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: a430f9ed84de3008ca5b632b1ed304bac0e753f6 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-phlogistonjohn-testing-2024-04-25-1915 sha1: a430f9ed84de3008ca5b632b1ed304bac0e753f6 owner: scheduled_phlogistonjohn@teuthology priority: 89 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 4852 sha1: a430f9ed84de3008ca5b632b1ed304bac0e753f6 sleep_before_teardown: 0 subset: 1/30 suite: orch:cephadm suite_branch: wip-phlogistonjohn-testing-2024-04-25-1915 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_a430f9ed84de3008ca5b632b1ed304bac0e753f6/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: a430f9ed84de3008ca5b632b1ed304bac0e753f6 targets: smithi112.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPEGdyFASQqIFDz81a7B1GoNomPeGMJxz1lXir8myDGuap85gzjtBqtiV6APEVZk6DGrxJ6QseJbeeY50yJsMas= smithi164.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGNs1y52GWnWZtIbW3HE65q4oxXRKIKsh477iEMNZ3WG13Aw8CmCl0SsuMaEghBsy2FOsPrwFqdNPZKwpfNGz0A= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-26_15:06:39 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: phlogistonjohn verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-04-26T15:16:03.198 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_a430f9ed84de3008ca5b632b1ed304bac0e753f6/qa; will attempt to use it 2024-04-26T15:16:03.199 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_a430f9ed84de3008ca5b632b1ed304bac0e753f6/qa/tasks 2024-04-26T15:16:03.199 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-26T15:16:03.201 INFO:teuthology.task.internal:Checking packages... 2024-04-26T15:16:03.224 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'a430f9ed84de3008ca5b632b1ed304bac0e753f6' 2024-04-26T15:16:03.224 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T15:16:03.224 INFO:teuthology.packaging:ref: None 2024-04-26T15:16:03.224 INFO:teuthology.packaging:tag: None 2024-04-26T15:16:03.224 INFO:teuthology.packaging:branch: wip-phlogistonjohn-testing-2024-04-25-1915 2024-04-26T15:16:03.224 INFO:teuthology.packaging:sha1: a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T15:16:03.224 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-phlogistonjohn-testing-2024-04-25-1915 2024-04-26T15:16:03.443 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3305.ga430f9ed 2024-04-26T15:16:03.445 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-26T15:16:03.451 INFO:teuthology.task.internal:no buildpackages task found 2024-04-26T15:16:03.451 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-26T15:16:03.475 INFO:teuthology.task.internal:Saving configuration 2024-04-26T15:16:03.488 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-26T15:16:03.494 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T15:16:03.519 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi112.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674785', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 15:10:58.152791', 'locked_by': 'scheduled_phlogistonjohn@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPEGdyFASQqIFDz81a7B1GoNomPeGMJxz1lXir8myDGuap85gzjtBqtiV6APEVZk6DGrxJ6QseJbeeY50yJsMas='} 2024-04-26T15:16:03.543 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi164.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674785', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 15:10:58.150817', 'locked_by': 'scheduled_phlogistonjohn@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGNs1y52GWnWZtIbW3HE65q4oxXRKIKsh477iEMNZ3WG13Aw8CmCl0SsuMaEghBsy2FOsPrwFqdNPZKwpfNGz0A='} 2024-04-26T15:16:03.543 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-26T15:16:03.549 INFO:teuthology.task.internal:roles: ubuntu@smithi112.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-04-26T15:16:03.549 INFO:teuthology.task.internal:roles: ubuntu@smithi164.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-04-26T15:16:03.549 INFO:teuthology.run_tasks:Running task console_log... 2024-04-26T15:16:03.613 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f9224df6f70>, signals=[15]) 2024-04-26T15:16:03.613 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-26T15:16:03.619 INFO:teuthology.task.internal:Opening connections... 2024-04-26T15:16:03.619 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:16:03.621 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi112.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T15:16:03.694 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi164.front.sepia.ceph.com 2024-04-26T15:16:03.695 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T15:16:03.767 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-26T15:16:03.774 DEBUG:teuthology.orchestra.run.smithi112:> uname -m 2024-04-26T15:16:03.791 INFO:teuthology.orchestra.run.smithi112.stdout:x86_64 2024-04-26T15:16:03.791 DEBUG:teuthology.orchestra.run.smithi112:> cat /etc/os-release 2024-04-26T15:16:03.848 INFO:teuthology.orchestra.run.smithi112.stdout:NAME="CentOS Stream" 2024-04-26T15:16:03.848 INFO:teuthology.orchestra.run.smithi112.stdout:VERSION="9" 2024-04-26T15:16:03.848 INFO:teuthology.orchestra.run.smithi112.stdout:ID="centos" 2024-04-26T15:16:03.848 INFO:teuthology.orchestra.run.smithi112.stdout:ID_LIKE="rhel fedora" 2024-04-26T15:16:03.848 INFO:teuthology.orchestra.run.smithi112.stdout:VERSION_ID="9" 2024-04-26T15:16:03.848 INFO:teuthology.orchestra.run.smithi112.stdout:PLATFORM_ID="platform:el9" 2024-04-26T15:16:03.848 INFO:teuthology.orchestra.run.smithi112.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T15:16:03.848 INFO:teuthology.orchestra.run.smithi112.stdout:ANSI_COLOR="0;31" 2024-04-26T15:16:03.848 INFO:teuthology.orchestra.run.smithi112.stdout:LOGO="fedora-logo-icon" 2024-04-26T15:16:03.849 INFO:teuthology.orchestra.run.smithi112.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T15:16:03.849 INFO:teuthology.orchestra.run.smithi112.stdout:HOME_URL="https://centos.org/" 2024-04-26T15:16:03.849 INFO:teuthology.orchestra.run.smithi112.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T15:16:03.849 INFO:teuthology.orchestra.run.smithi112.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T15:16:03.849 INFO:teuthology.orchestra.run.smithi112.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T15:16:03.849 INFO:teuthology.lock.ops:Updating smithi112.front.sepia.ceph.com on lock server 2024-04-26T15:16:03.874 DEBUG:teuthology.orchestra.run.smithi164:> uname -m 2024-04-26T15:16:03.891 INFO:teuthology.orchestra.run.smithi164.stdout:x86_64 2024-04-26T15:16:03.892 DEBUG:teuthology.orchestra.run.smithi164:> cat /etc/os-release 2024-04-26T15:16:03.948 INFO:teuthology.orchestra.run.smithi164.stdout:NAME="CentOS Stream" 2024-04-26T15:16:03.948 INFO:teuthology.orchestra.run.smithi164.stdout:VERSION="9" 2024-04-26T15:16:03.948 INFO:teuthology.orchestra.run.smithi164.stdout:ID="centos" 2024-04-26T15:16:03.948 INFO:teuthology.orchestra.run.smithi164.stdout:ID_LIKE="rhel fedora" 2024-04-26T15:16:03.948 INFO:teuthology.orchestra.run.smithi164.stdout:VERSION_ID="9" 2024-04-26T15:16:03.949 INFO:teuthology.orchestra.run.smithi164.stdout:PLATFORM_ID="platform:el9" 2024-04-26T15:16:03.949 INFO:teuthology.orchestra.run.smithi164.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T15:16:03.949 INFO:teuthology.orchestra.run.smithi164.stdout:ANSI_COLOR="0;31" 2024-04-26T15:16:03.949 INFO:teuthology.orchestra.run.smithi164.stdout:LOGO="fedora-logo-icon" 2024-04-26T15:16:03.949 INFO:teuthology.orchestra.run.smithi164.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T15:16:03.949 INFO:teuthology.orchestra.run.smithi164.stdout:HOME_URL="https://centos.org/" 2024-04-26T15:16:03.949 INFO:teuthology.orchestra.run.smithi164.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T15:16:03.949 INFO:teuthology.orchestra.run.smithi164.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T15:16:03.949 INFO:teuthology.orchestra.run.smithi164.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T15:16:03.949 INFO:teuthology.lock.ops:Updating smithi164.front.sepia.ceph.com on lock server 2024-04-26T15:16:03.974 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-26T15:16:03.983 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-26T15:16:03.988 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-26T15:16:03.988 DEBUG:teuthology.orchestra.run.smithi112:> test '!' -e /home/ubuntu/cephtest 2024-04-26T15:16:03.991 DEBUG:teuthology.orchestra.run.smithi164:> test '!' -e /home/ubuntu/cephtest 2024-04-26T15:16:04.006 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-26T15:16:04.012 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-26T15:16:04.013 DEBUG:teuthology.orchestra.run.smithi112:> test -z $(ls -A /var/lib/ceph) 2024-04-26T15:16:04.047 DEBUG:teuthology.orchestra.run.smithi164:> test -z $(ls -A /var/lib/ceph) 2024-04-26T15:16:04.133 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-26T15:16:04.191 INFO:teuthology.run_tasks:Running task kernel... 2024-04-26T15:16:04.205 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-04-26T15:16:04.205 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-04-26T15:16:04.205 DEBUG:teuthology.orchestra.run.smithi112:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T15:16:04.206 DEBUG:teuthology.orchestra.run.smithi164:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T15:16:04.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T15:16:04.222 DEBUG:teuthology.orchestra.run.smithi112:> uname -r 2024-04-26T15:16:04.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T15:16:04.223 DEBUG:teuthology.orchestra.run.smithi164:> uname -r 2024-04-26T15:16:04.279 INFO:teuthology.orchestra.run.smithi112.stdout:5.14.0-437.el9.x86_64 2024-04-26T15:16:04.279 INFO:teuthology.task.kernel:Running kernel on smithi112: 5.14.0-437.el9.x86_64 2024-04-26T15:16:04.279 DEBUG:teuthology.orchestra.run.smithi112:> sudo yum install -y kernel 2024-04-26T15:16:04.280 INFO:teuthology.orchestra.run.smithi164.stdout:5.14.0-437.el9.x86_64 2024-04-26T15:16:04.280 INFO:teuthology.task.kernel:Running kernel on smithi164: 5.14.0-437.el9.x86_64 2024-04-26T15:16:04.281 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum install -y kernel 2024-04-26T15:16:06.798 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - BaseOS 8.2 MB/s | 8.0 MB 00:00 2024-04-26T15:16:06.938 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS Stream 9 - BaseOS 7.4 MB/s | 8.0 MB 00:01 2024-04-26T15:16:08.924 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - AppStream 18 MB/s | 19 MB 00:01 2024-04-26T15:16:09.422 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS Stream 9 - AppStream 13 MB/s | 19 MB 00:01 2024-04-26T15:16:13.417 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - CRB 6.5 MB/s | 6.2 MB 00:00 2024-04-26T15:16:13.582 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.2 MB 00:00 2024-04-26T15:16:14.971 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS Stream 9 - Extras packages 66 kB/s | 16 kB 00:00 2024-04-26T15:16:15.073 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - Extras packages 32 kB/s | 16 kB 00:00 2024-04-26T15:16:16.131 INFO:teuthology.orchestra.run.smithi112.stdout:Extra Packages for Enterprise Linux 20 MB/s | 21 MB 00:01 2024-04-26T15:16:17.277 INFO:teuthology.orchestra.run.smithi164.stdout:Extra Packages for Enterprise Linux 10 MB/s | 21 MB 00:02 2024-04-26T15:16:21.122 INFO:teuthology.orchestra.run.smithi112.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-04-26T15:16:22.366 INFO:teuthology.orchestra.run.smithi164.stdout:lab-extras 12 kB/s | 1.7 kB 00:00 2024-04-26T15:16:22.945 INFO:teuthology.orchestra.run.smithi112.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T15:16:22.945 INFO:teuthology.orchestra.run.smithi112.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T15:16:23.032 INFO:teuthology.orchestra.run.smithi112.stdout:Dependencies resolved. 2024-04-26T15:16:23.032 INFO:teuthology.orchestra.run.smithi112.stdout:Nothing to do. 2024-04-26T15:16:23.032 INFO:teuthology.orchestra.run.smithi112.stdout:Complete! 2024-04-26T15:16:23.143 DEBUG:teuthology.orchestra.run.smithi112:> echo no | sudo yum reinstall kernel || true 2024-04-26T15:16:23.641 INFO:teuthology.orchestra.run.smithi112.stdout:Last metadata expiration check: 0:00:02 ago on Fri 26 Apr 2024 03:16:21 PM UTC. 2024-04-26T15:16:23.760 INFO:teuthology.orchestra.run.smithi112.stdout:Dependencies resolved. 2024-04-26T15:16:23.778 INFO:teuthology.orchestra.run.smithi112.stdout:================================================================================ 2024-04-26T15:16:23.778 INFO:teuthology.orchestra.run.smithi112.stdout: Package Architecture Version Repository Size 2024-04-26T15:16:23.778 INFO:teuthology.orchestra.run.smithi112.stdout:================================================================================ 2024-04-26T15:16:23.778 INFO:teuthology.orchestra.run.smithi112.stdout:Reinstalling: 2024-04-26T15:16:23.779 INFO:teuthology.orchestra.run.smithi112.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T15:16:23.779 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:16:23.779 INFO:teuthology.orchestra.run.smithi112.stdout:Transaction Summary 2024-04-26T15:16:23.779 INFO:teuthology.orchestra.run.smithi112.stdout:================================================================================ 2024-04-26T15:16:23.779 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:16:23.779 INFO:teuthology.orchestra.run.smithi112.stdout:Total download size: 255 k 2024-04-26T15:16:23.779 INFO:teuthology.orchestra.run.smithi112.stdout:Installed size: 0 2024-04-26T15:16:23.779 INFO:teuthology.orchestra.run.smithi112.stderr:Operation aborted. 2024-04-26T15:16:23.831 INFO:teuthology.orchestra.run.smithi112.stdout:Is this ok [y/N]: 2024-04-26T15:16:23.831 DEBUG:teuthology.orchestra.run.smithi112:> sudo yum reinstall -y kernel || true 2024-04-26T15:16:24.264 INFO:teuthology.orchestra.run.smithi164.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T15:16:24.264 INFO:teuthology.orchestra.run.smithi164.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T15:16:24.319 INFO:teuthology.orchestra.run.smithi112.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 03:16:21 PM UTC. 2024-04-26T15:16:24.351 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-04-26T15:16:24.352 INFO:teuthology.orchestra.run.smithi164.stdout:Nothing to do. 2024-04-26T15:16:24.352 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-04-26T15:16:24.434 INFO:teuthology.orchestra.run.smithi112.stdout:Dependencies resolved. 2024-04-26T15:16:24.435 INFO:teuthology.orchestra.run.smithi112.stdout:================================================================================ 2024-04-26T15:16:24.435 INFO:teuthology.orchestra.run.smithi112.stdout: Package Architecture Version Repository Size 2024-04-26T15:16:24.435 INFO:teuthology.orchestra.run.smithi112.stdout:================================================================================ 2024-04-26T15:16:24.435 INFO:teuthology.orchestra.run.smithi112.stdout:Reinstalling: 2024-04-26T15:16:24.435 INFO:teuthology.orchestra.run.smithi112.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T15:16:24.435 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:16:24.435 INFO:teuthology.orchestra.run.smithi112.stdout:Transaction Summary 2024-04-26T15:16:24.435 INFO:teuthology.orchestra.run.smithi112.stdout:================================================================================ 2024-04-26T15:16:24.435 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:16:24.435 INFO:teuthology.orchestra.run.smithi112.stdout:Total download size: 255 k 2024-04-26T15:16:24.435 INFO:teuthology.orchestra.run.smithi112.stdout:Installed size: 0 2024-04-26T15:16:24.435 INFO:teuthology.orchestra.run.smithi112.stdout:Downloading Packages: 2024-04-26T15:16:24.464 DEBUG:teuthology.orchestra.run.smithi164:> echo no | sudo yum reinstall kernel || true 2024-04-26T15:16:24.642 INFO:teuthology.orchestra.run.smithi112.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.8 MB/s | 255 kB 00:00 2024-04-26T15:16:24.643 INFO:teuthology.orchestra.run.smithi112.stdout:-------------------------------------------------------------------------------- 2024-04-26T15:16:24.643 INFO:teuthology.orchestra.run.smithi112.stdout:Total 1.2 MB/s | 255 kB 00:00 2024-04-26T15:16:24.664 INFO:teuthology.orchestra.run.smithi112.stdout:Running transaction check 2024-04-26T15:16:24.849 INFO:teuthology.orchestra.run.smithi112.stdout:Transaction check succeeded. 2024-04-26T15:16:24.849 INFO:teuthology.orchestra.run.smithi112.stdout:Running transaction test 2024-04-26T15:16:24.854 INFO:teuthology.orchestra.run.smithi112.stdout:Transaction test succeeded. 2024-04-26T15:16:24.854 INFO:teuthology.orchestra.run.smithi112.stdout:Running transaction 2024-04-26T15:16:24.961 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:00:02 ago on Fri 26 Apr 2024 03:16:22 PM UTC. 2024-04-26T15:16:24.991 INFO:teuthology.orchestra.run.smithi112.stdout: Preparing : 1/1 2024-04-26T15:16:25.059 INFO:teuthology.orchestra.run.smithi112.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T15:16:25.060 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T15:16:25.088 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-04-26T15:16:25.099 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-04-26T15:16:25.099 INFO:teuthology.orchestra.run.smithi164.stdout: Package Architecture Version Repository Size 2024-04-26T15:16:25.099 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-04-26T15:16:25.099 INFO:teuthology.orchestra.run.smithi164.stdout:Reinstalling: 2024-04-26T15:16:25.099 INFO:teuthology.orchestra.run.smithi164.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T15:16:25.099 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:16:25.099 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-04-26T15:16:25.100 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-04-26T15:16:25.100 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:16:25.100 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 255 k 2024-04-26T15:16:25.100 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 0 2024-04-26T15:16:25.100 INFO:teuthology.orchestra.run.smithi164.stderr:Operation aborted. 2024-04-26T15:16:25.158 INFO:teuthology.orchestra.run.smithi164.stdout:Is this ok [y/N]: 2024-04-26T15:16:25.159 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum reinstall -y kernel || true 2024-04-26T15:16:25.650 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 03:16:22 PM UTC. 2024-04-26T15:16:25.702 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T15:16:25.702 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T15:16:25.766 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-04-26T15:16:25.767 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-04-26T15:16:25.767 INFO:teuthology.orchestra.run.smithi164.stdout: Package Architecture Version Repository Size 2024-04-26T15:16:25.767 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-04-26T15:16:25.767 INFO:teuthology.orchestra.run.smithi164.stdout:Reinstalling: 2024-04-26T15:16:25.767 INFO:teuthology.orchestra.run.smithi164.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T15:16:25.767 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:16:25.768 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-04-26T15:16:25.768 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-04-26T15:16:25.768 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:16:25.768 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 255 k 2024-04-26T15:16:25.768 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 0 2024-04-26T15:16:25.768 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-04-26T15:16:26.143 INFO:teuthology.orchestra.run.smithi164.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.1 MB/s | 255 kB 00:00 2024-04-26T15:16:26.143 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-04-26T15:16:26.143 INFO:teuthology.orchestra.run.smithi164.stdout:Total 676 kB/s | 255 kB 00:00 2024-04-26T15:16:26.159 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-04-26T15:16:26.338 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-04-26T15:16:26.339 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-04-26T15:16:26.343 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-04-26T15:16:26.343 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-04-26T15:16:26.430 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T15:16:26.431 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:16:26.431 INFO:teuthology.orchestra.run.smithi112.stdout:Reinstalled: 2024-04-26T15:16:26.431 INFO:teuthology.orchestra.run.smithi112.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T15:16:26.431 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:16:26.431 INFO:teuthology.orchestra.run.smithi112.stdout:Complete! 2024-04-26T15:16:26.469 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-04-26T15:16:26.560 INFO:teuthology.orchestra.run.smithi164.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T15:16:26.561 INFO:teuthology.orchestra.run.smithi164.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T15:16:26.620 DEBUG:teuthology.orchestra.run.smithi112:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T15:16:26.666 INFO:teuthology.orchestra.run.smithi112.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T15:16:26.667 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T15:16:26.667 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T15:16:26.667 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-04-26T15:16:26.667 DEBUG:teuthology.orchestra.run.smithi112:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T15:16:26.700 INFO:teuthology.orchestra.run.smithi112.stdout:ttyS1 2024-04-26T15:16:26.721 DEBUG:teuthology.parallel:result is None 2024-04-26T15:16:27.259 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T15:16:27.259 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T15:16:27.948 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T15:16:27.948 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:16:27.948 INFO:teuthology.orchestra.run.smithi164.stdout:Reinstalled: 2024-04-26T15:16:27.948 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T15:16:27.948 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:16:27.948 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-04-26T15:16:28.164 DEBUG:teuthology.orchestra.run.smithi164:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T15:16:28.212 INFO:teuthology.orchestra.run.smithi164.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T15:16:28.212 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T15:16:28.212 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T15:16:28.212 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-04-26T15:16:28.213 DEBUG:teuthology.orchestra.run.smithi164:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T15:16:28.244 INFO:teuthology.orchestra.run.smithi164.stdout:ttyS1 2024-04-26T15:16:28.266 DEBUG:teuthology.parallel:result is None 2024-04-26T15:16:28.267 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-26T15:16:28.274 INFO:teuthology.task.internal:Creating test directory... 2024-04-26T15:16:28.274 DEBUG:teuthology.orchestra.run.smithi112:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T15:16:28.277 DEBUG:teuthology.orchestra.run.smithi164:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T15:16:28.321 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-26T15:16:28.328 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-26T15:16:28.334 INFO:teuthology.task.internal:Creating archive directory... 2024-04-26T15:16:28.334 DEBUG:teuthology.orchestra.run.smithi112:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T15:16:28.336 DEBUG:teuthology.orchestra.run.smithi164:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T15:16:28.391 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-26T15:16:28.398 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-26T15:16:28.398 DEBUG:teuthology.orchestra.run.smithi112:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T15:16:28.411 DEBUG:teuthology.orchestra.run.smithi164:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T15:16:28.444 INFO:teuthology.orchestra.run.smithi112.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T15:16:28.455 INFO:teuthology.orchestra.run.smithi112.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T15:16:28.460 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T15:16:28.469 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T15:16:28.471 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-26T15:16:28.478 INFO:teuthology.task.internal:Configuring sudo... 2024-04-26T15:16:28.478 DEBUG:teuthology.orchestra.run.smithi112:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T15:16:28.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T15:16:28.534 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-26T15:16:28.543 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-26T15:16:28.543 DEBUG:teuthology.orchestra.run.smithi112:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T15:16:28.566 DEBUG:teuthology.orchestra.run.smithi164:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T15:16:28.589 DEBUG:teuthology.orchestra.run.smithi112:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T15:16:28.667 DEBUG:teuthology.orchestra.run.smithi112:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T15:16:28.709 DEBUG:teuthology.orchestra.run.smithi112:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T15:16:28.791 DEBUG:teuthology.orchestra.run.smithi112:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T15:16:28.832 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-26T15:16:28.832 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T15:16:28.898 DEBUG:teuthology.orchestra.run.smithi164:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T15:16:28.938 DEBUG:teuthology.orchestra.run.smithi164:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T15:16:28.987 DEBUG:teuthology.orchestra.run.smithi164:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T15:16:29.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T15:16:29.107 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-04-26T15:16:29.108 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T15:16:29.171 DEBUG:teuthology.orchestra.run.smithi112:> sudo service rsyslog restart 2024-04-26T15:16:29.174 DEBUG:teuthology.orchestra.run.smithi164:> sudo service rsyslog restart 2024-04-26T15:16:29.208 INFO:teuthology.orchestra.run.smithi112.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T15:16:29.255 INFO:teuthology.orchestra.run.smithi164.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T15:16:29.639 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-26T15:16:29.647 INFO:teuthology.task.internal:Starting timer... 2024-04-26T15:16:29.647 INFO:teuthology.run_tasks:Running task pcp... 2024-04-26T15:16:29.655 INFO:teuthology.run_tasks:Running task selinux... 2024-04-26T15:16:29.663 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-04-26T15:16:29.664 DEBUG:teuthology.orchestra.run.smithi112:> sudo service auditd rotate 2024-04-26T15:16:29.715 INFO:teuthology.orchestra.run.smithi112.stdout:Rotating logs: 2024-04-26T15:16:29.717 DEBUG:teuthology.orchestra.run.smithi164:> sudo service auditd rotate 2024-04-26T15:16:29.791 INFO:teuthology.orchestra.run.smithi164.stdout:Rotating logs: 2024-04-26T15:16:29.793 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-26T15:16:29.794 DEBUG:teuthology.orchestra.run.smithi112:> /usr/sbin/getenforce 2024-04-26T15:16:29.816 INFO:teuthology.orchestra.run.smithi112.stdout:Permissive 2024-04-26T15:16:29.816 DEBUG:teuthology.orchestra.run.smithi164:> /usr/sbin/getenforce 2024-04-26T15:16:29.839 INFO:teuthology.orchestra.run.smithi164.stdout:Permissive 2024-04-26T15:16:29.839 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi112.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi164.front.sepia.ceph.com': 'permissive'} 2024-04-26T15:16:29.840 DEBUG:teuthology.orchestra.run.smithi112:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-26T15:16:29.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T15:16:29.882 DEBUG:teuthology.orchestra.run.smithi164:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-26T15:16:29.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T15:16:29.907 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-26T15:16:29.907 DEBUG:teuthology.orchestra.run.smithi112:> sudo /usr/sbin/setenforce permissive 2024-04-26T15:16:29.946 DEBUG:teuthology.orchestra.run.smithi164:> sudo /usr/sbin/setenforce permissive 2024-04-26T15:16:29.971 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-26T15:16:29.981 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-26T15:16:29.996 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-26T15:16:29.997 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi112.front.sepia.ceph.com,smithi164.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-26T15:19:58.400 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi112.front.sepia.ceph.com'), Remote(name='ubuntu@smithi164.front.sepia.ceph.com')] 2024-04-26T15:19:58.401 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi112.front.sepia.ceph.com' 2024-04-26T15:19:58.402 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi112.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T15:19:58.475 DEBUG:teuthology.orchestra.run.smithi112:> true 2024-04-26T15:19:58.557 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi112.front.sepia.ceph.com' 2024-04-26T15:19:58.557 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-04-26T15:19:58.558 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T15:19:58.629 DEBUG:teuthology.orchestra.run.smithi164:> true 2024-04-26T15:19:58.705 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-04-26T15:19:58.705 INFO:teuthology.run_tasks:Running task clock... 2024-04-26T15:19:58.716 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-26T15:19:58.717 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T15:19:58.717 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T15:19:58.719 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T15:19:58.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T15:19:58.745 INFO:teuthology.orchestra.run.smithi112.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T15:19:58.758 INFO:teuthology.orchestra.run.smithi112.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T15:19:58.772 INFO:teuthology.orchestra.run.smithi164.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T15:19:58.780 INFO:teuthology.orchestra.run.smithi112.stderr:sudo: ntpd: command not found 2024-04-26T15:19:58.785 INFO:teuthology.orchestra.run.smithi164.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T15:19:58.790 INFO:teuthology.orchestra.run.smithi112.stdout:506 Cannot talk to daemon 2024-04-26T15:19:58.803 INFO:teuthology.orchestra.run.smithi112.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T15:19:58.808 INFO:teuthology.orchestra.run.smithi164.stderr:sudo: ntpd: command not found 2024-04-26T15:19:58.815 INFO:teuthology.orchestra.run.smithi112.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T15:19:58.818 INFO:teuthology.orchestra.run.smithi164.stdout:506 Cannot talk to daemon 2024-04-26T15:19:58.831 INFO:teuthology.orchestra.run.smithi164.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T15:19:58.844 INFO:teuthology.orchestra.run.smithi164.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T15:19:58.869 INFO:teuthology.orchestra.run.smithi112.stderr:bash: line 1: ntpq: command not found 2024-04-26T15:19:58.873 INFO:teuthology.orchestra.run.smithi112.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T15:19:58.873 INFO:teuthology.orchestra.run.smithi112.stdout:=============================================================================== 2024-04-26T15:19:58.873 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:19:58.873 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:19:58.873 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:19:58.873 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:19:58.897 INFO:teuthology.orchestra.run.smithi164.stderr:bash: line 1: ntpq: command not found 2024-04-26T15:19:58.901 INFO:teuthology.orchestra.run.smithi164.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T15:19:58.901 INFO:teuthology.orchestra.run.smithi164.stdout:=============================================================================== 2024-04-26T15:19:58.901 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:19:58.901 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:19:58.901 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:19:58.901 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:19:58.901 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-26T15:19:58.999 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': 'a430f9ed84de3008ca5b632b1ed304bac0e753f6'} 2024-04-26T15:19:59.000 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-26T15:19:59.000 INFO:tasks.cephadm:Cluster fsid is 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:19:59.000 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-26T15:19:59.000 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-04-26T15:19:59.000 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi112': '172.21.15.112', 'mon.smithi164': '172.21.15.164'} 2024-04-26T15:19:59.001 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-26T15:19:59.001 DEBUG:teuthology.orchestra.run.smithi112:> sudo hostname $(hostname -s) 2024-04-26T15:19:59.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo hostname $(hostname -s) 2024-04-26T15:19:59.050 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-26T15:19:59.050 DEBUG:teuthology.orchestra.run.smithi112:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T15:19:59.324 INFO:teuthology.orchestra.run.smithi112.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 26 15:19 /home/ubuntu/cephtest/cephadm 2024-04-26T15:19:59.324 DEBUG:teuthology.orchestra.run.smithi164:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T15:19:59.576 INFO:teuthology.orchestra.run.smithi164.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 26 15:19 /home/ubuntu/cephtest/cephadm 2024-04-26T15:19:59.576 DEBUG:teuthology.orchestra.run.smithi112:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T15:19:59.592 DEBUG:teuthology.orchestra.run.smithi164:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T15:19:59.616 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-26T15:19:59.617 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-26T15:19:59.634 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-26T15:20:01.113 INFO:teuthology.orchestra.run.smithi164.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-26T15:20:01.184 INFO:teuthology.orchestra.run.smithi112.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-26T15:20:20.510 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2024-04-26T15:20:20.510 INFO:teuthology.orchestra.run.smithi164.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-26T15:20:20.510 INFO:teuthology.orchestra.run.smithi164.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-26T15:20:20.511 INFO:teuthology.orchestra.run.smithi164.stdout: "repo_digests": [ 2024-04-26T15:20:20.511 INFO:teuthology.orchestra.run.smithi164.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-26T15:20:20.511 INFO:teuthology.orchestra.run.smithi164.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-26T15:20:20.511 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2024-04-26T15:20:20.511 INFO:teuthology.orchestra.run.smithi164.stdout:} 2024-04-26T15:20:20.730 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:20:20.730 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-26T15:20:20.730 INFO:teuthology.orchestra.run.smithi112.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-26T15:20:20.730 INFO:teuthology.orchestra.run.smithi112.stdout: "repo_digests": [ 2024-04-26T15:20:20.730 INFO:teuthology.orchestra.run.smithi112.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-26T15:20:20.731 INFO:teuthology.orchestra.run.smithi112.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-26T15:20:20.731 INFO:teuthology.orchestra.run.smithi112.stdout: ] 2024-04-26T15:20:20.731 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:20:20.748 DEBUG:teuthology.orchestra.run.smithi112:> sudo mkdir -p /etc/ceph 2024-04-26T15:20:20.797 DEBUG:teuthology.orchestra.run.smithi164:> sudo mkdir -p /etc/ceph 2024-04-26T15:20:20.825 DEBUG:teuthology.orchestra.run.smithi112:> sudo chmod 777 /etc/ceph 2024-04-26T15:20:20.848 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 777 /etc/ceph 2024-04-26T15:20:20.888 INFO:tasks.cephadm:Writing seed config... 2024-04-26T15:20:20.889 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-26T15:20:20.889 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-26T15:20:20.889 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-26T15:20:20.889 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-26T15:20:20.889 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-26T15:20:20.889 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-26T15:20:20.889 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-26T15:20:20.889 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-26T15:20:20.890 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-26T15:20:20.890 DEBUG:teuthology.orchestra.run.smithi112:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-26T15:20:20.905 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 71e28ad8-03e0-11ef-bc93-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-26T15:20:20.905 DEBUG:teuthology.orchestra.run.smithi112:mon.smithi112> sudo journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi112.service 2024-04-26T15:20:20.947 INFO:tasks.cephadm:Bootstrapping... 2024-04-26T15:20:20.947 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.112 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-26T15:20:21.138 INFO:teuthology.orchestra.run.smithi112.stderr:-------------------------------------------------------------------------------- 2024-04-26T15:20:21.138 INFO:teuthology.orchestra.run.smithi112.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '71e28ad8-03e0-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.112', '--skip-admin-label'] 2024-04-26T15:20:21.161 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: 4.9.4-dev 2024-04-26T15:20:21.163 INFO:teuthology.orchestra.run.smithi112.stderr:Verifying podman|docker is present... 2024-04-26T15:20:21.185 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: 4.9.4-dev 2024-04-26T15:20:21.187 INFO:teuthology.orchestra.run.smithi112.stderr:Verifying lvm2 is present... 2024-04-26T15:20:21.187 INFO:teuthology.orchestra.run.smithi112.stderr:Verifying time synchronization is in place... 2024-04-26T15:20:21.194 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T15:20:21.200 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: inactive 2024-04-26T15:20:21.207 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: enabled 2024-04-26T15:20:21.212 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: active 2024-04-26T15:20:21.213 INFO:teuthology.orchestra.run.smithi112.stderr:Unit chronyd.service is enabled and running 2024-04-26T15:20:21.213 INFO:teuthology.orchestra.run.smithi112.stderr:Repeating the final host check... 2024-04-26T15:20:21.234 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: 4.9.4-dev 2024-04-26T15:20:21.236 INFO:teuthology.orchestra.run.smithi112.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-26T15:20:21.237 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl is present 2024-04-26T15:20:21.237 INFO:teuthology.orchestra.run.smithi112.stderr:lvcreate is present 2024-04-26T15:20:21.242 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T15:20:21.248 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: inactive 2024-04-26T15:20:21.255 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: enabled 2024-04-26T15:20:21.261 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: active 2024-04-26T15:20:21.261 INFO:teuthology.orchestra.run.smithi112.stderr:Unit chronyd.service is enabled and running 2024-04-26T15:20:21.262 INFO:teuthology.orchestra.run.smithi112.stderr:Host looks OK 2024-04-26T15:20:21.262 INFO:teuthology.orchestra.run.smithi112.stderr:Cluster fsid: 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:20:21.262 INFO:teuthology.orchestra.run.smithi112.stderr:Acquiring lock 139671144707360 on /run/cephadm/71e28ad8-03e0-11ef-bc93-c7b262605968.lock 2024-04-26T15:20:21.262 INFO:teuthology.orchestra.run.smithi112.stderr:Lock 139671144707360 acquired on /run/cephadm/71e28ad8-03e0-11ef-bc93-c7b262605968.lock 2024-04-26T15:20:21.262 INFO:teuthology.orchestra.run.smithi112.stderr:Verifying IP 172.21.15.112 port 3300 ... 2024-04-26T15:20:21.263 INFO:teuthology.orchestra.run.smithi112.stderr:Verifying IP 172.21.15.112 port 6789 ... 2024-04-26T15:20:21.263 INFO:teuthology.orchestra.run.smithi112.stderr:Base mon IP is 172.21.15.112, final addrv is [v2:172.21.15.112:3300,v1:172.21.15.112:6789] 2024-04-26T15:20:21.265 INFO:teuthology.orchestra.run.smithi112.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.112 metric 100 2024-04-26T15:20:21.265 INFO:teuthology.orchestra.run.smithi112.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.112 metric 100 2024-04-26T15:20:21.267 INFO:teuthology.orchestra.run.smithi112.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-26T15:20:21.267 INFO:teuthology.orchestra.run.smithi112.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-04-26T15:20:21.267 INFO:teuthology.orchestra.run.smithi112.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-04-26T15:20:21.272 INFO:teuthology.orchestra.run.smithi112.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-26T15:20:21.272 INFO:teuthology.orchestra.run.smithi112.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-26T15:20:21.272 INFO:teuthology.orchestra.run.smithi112.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-26T15:20:21.272 INFO:teuthology.orchestra.run.smithi112.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-26T15:20:21.272 INFO:teuthology.orchestra.run.smithi112.stderr:/sbin/ip: inet6 fe80::a649:7080:fd3b:421a/64 scope link noprefixroute 2024-04-26T15:20:21.272 INFO:teuthology.orchestra.run.smithi112.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-26T15:20:21.272 INFO:teuthology.orchestra.run.smithi112.stderr:Mon IP `172.21.15.112` is in CIDR network `172.21.0.0/20` 2024-04-26T15:20:21.272 INFO:teuthology.orchestra.run.smithi112.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-26T15:20:21.272 INFO:teuthology.orchestra.run.smithi112.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-26T15:20:21.373 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-26T15:20:21.823 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: Getting image source signatures 2024-04-26T15:20:21.823 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-26T15:20:21.823 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-26T15:20:21.823 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-26T15:20:21.823 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-26T15:20:21.823 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-26T15:20:21.823 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-26T15:20:21.975 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: Writing manifest to image destination 2024-04-26T15:20:22.136 INFO:teuthology.orchestra.run.smithi112.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-26T15:20:25.604 INFO:teuthology.orchestra.run.smithi112.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-26T15:20:31.195 INFO:teuthology.orchestra.run.smithi112.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-26T15:20:31.195 INFO:teuthology.orchestra.run.smithi112.stderr:Extracting ceph user uid/gid from container image... 2024-04-26T15:20:32.454 INFO:teuthology.orchestra.run.smithi112.stderr:stat: 167 167 2024-04-26T15:20:32.993 INFO:teuthology.orchestra.run.smithi112.stderr:Creating initial keys... 2024-04-26T15:20:33.410 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph-authtool: AQBBxitm2llxExAAJ7wEeGvwo//m0WSiVkfKeQ== 2024-04-26T15:20:34.266 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph-authtool: AQBCxitmjzHSChAADLG0cpDrthOrDzv/XsGm0w== 2024-04-26T15:20:35.246 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph-authtool: AQBDxitmb4OjCRAA+9IlbwscFCH0pm2FUHVXlg== 2024-04-26T15:20:35.803 INFO:teuthology.orchestra.run.smithi112.stderr:Creating initial monmap... 2024-04-26T15:20:36.210 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T15:20:36.211 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-26T15:20:36.211 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:20:36.211 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T15:20:37.375 INFO:teuthology.orchestra.run.smithi112.stderr:monmaptool for smithi112 [v2:172.21.15.112:3300,v1:172.21.15.112:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T15:20:37.375 INFO:teuthology.orchestra.run.smithi112.stderr:setting min_mon_release = octopus 2024-04-26T15:20:37.376 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/monmaptool: set fsid to 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:20:37.376 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T15:20:37.376 INFO:teuthology.orchestra.run.smithi112.stderr: 2024-04-26T15:20:37.376 INFO:teuthology.orchestra.run.smithi112.stderr:Creating mon... 2024-04-26T15:20:39.584 INFO:teuthology.orchestra.run.smithi112.stderr:create mon.smithi112 on 2024-04-26T15:20:39.783 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-26T15:20:39.927 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-71e28ad8-03e0-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-71e28ad8-03e0-11ef-bc93-c7b262605968.target. 2024-04-26T15:20:39.927 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-71e28ad8-03e0-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-71e28ad8-03e0-11ef-bc93-c7b262605968.target. 2024-04-26T15:20:40.219 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: Failed to reset failed state of unit ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi112.service: Unit ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi112.service not loaded. 2024-04-26T15:20:40.227 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: Created symlink /etc/systemd/system/ceph-71e28ad8-03e0-11ef-bc93-c7b262605968.target.wants/ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi112.service → /etc/systemd/system/ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@.service. 2024-04-26T15:20:40.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:40 smithi112 podman[21419]: 2024-04-26 15:20:40.847487926 +0000 UTC m=+0.326431106 container start 29e45d1fb1b5bf6083a870c0f881403fdd7fa5d966dab73740d99072013b0946 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, RELEASE=HEAD, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, ceph=True, name=centos-stream, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., version=8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8) 2024-04-26T15:20:40.931 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: disabled 2024-04-26T15:20:40.937 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: inactive 2024-04-26T15:20:40.938 INFO:teuthology.orchestra.run.smithi112.stderr:firewalld.service is not enabled 2024-04-26T15:20:40.938 INFO:teuthology.orchestra.run.smithi112.stderr:Not possible to enable service . firewalld.service is not available 2024-04-26T15:20:40.938 INFO:teuthology.orchestra.run.smithi112.stderr:Waiting for mon to start... 2024-04-26T15:20:40.939 INFO:teuthology.orchestra.run.smithi112.stderr:Waiting for mon... 2024-04-26T15:20:41.163 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:40 smithi112 ceph-mon[21438]: mkfs 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:20:41.163 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:40 smithi112 ceph-mon[21438]: mon.smithi112 is new leader, mons smithi112 in quorum (ranks 0) 2024-04-26T15:20:41.163 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:40 smithi112 bash[21419]: 29e45d1fb1b5bf6083a870c0f881403fdd7fa5d966dab73740d99072013b0946 2024-04-26T15:20:41.163 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:40 smithi112 systemd[1]: Started Ceph mon.smithi112 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: cluster: 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: id: 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: services: 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi112 (age 0.560091s) 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: data: 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-26T15:20:41.444 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: pgs: 2024-04-26T15:20:41.445 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:41.967 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:41 smithi112 ceph-mon[21438]: mon.smithi112 is new leader, mons smithi112 in quorum (ranks 0) 2024-04-26T15:20:41.967 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:41 smithi112 ceph-mon[21438]: monmap e1: 1 mons at {smithi112=[v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0]} 2024-04-26T15:20:41.967 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:41 smithi112 ceph-mon[21438]: fsmap 2024-04-26T15:20:41.967 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:41 smithi112 ceph-mon[21438]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T15:20:41.967 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:41 smithi112 ceph-mon[21438]: mgrmap e1: no daemons active 2024-04-26T15:20:41.967 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:41 smithi112 ceph-mon[21438]: from='client.? 172.21.15.112:0/1817656934' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T15:20:41.970 INFO:teuthology.orchestra.run.smithi112.stderr:mon is available 2024-04-26T15:20:41.970 INFO:teuthology.orchestra.run.smithi112.stderr:Assimilating anything we can from ceph.conf... 2024-04-26T15:20:42.480 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:42.480 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: [global] 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: fsid = 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.112:3300,v1:172.21.15.112:6789] 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: [mgr] 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: [osd] 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-26T15:20:42.481 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-26T15:20:43.082 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:42 smithi112 ceph-mon[21438]: from='client.? 172.21.15.112:0/1327607176' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T15:20:43.082 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:42 smithi112 ceph-mon[21438]: from='client.? 172.21.15.112:0/1327607176' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-26T15:20:43.084 INFO:teuthology.orchestra.run.smithi112.stderr:Generating new minimal ceph.conf... 2024-04-26T15:20:44.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:43 smithi112 ceph-mon[21438]: from='client.? 172.21.15.112:0/3338703453' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:20:44.459 INFO:teuthology.orchestra.run.smithi112.stderr:Restarting the monitor... 2024-04-26T15:20:44.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:44 smithi112 systemd[1]: Stopping Ceph mon.smithi112 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:20:44.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:44 smithi112 bash[21654]: Error: no container with name or ID "ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon.smithi112" found: no such container 2024-04-26T15:20:44.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112[21434]: 2024-04-26T15:20:44.625+0000 7f959b995700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi112 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:20:44.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112[21434]: 2024-04-26T15:20:44.625+0000 7f959b995700 -1 mon.smithi112@0(leader) e1 *** Got Signal Terminated *** 2024-04-26T15:20:44.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:44 smithi112 podman[21667]: 2024-04-26 15:20:44.656906684 +0000 UTC m=+0.139042706 container died 29e45d1fb1b5bf6083a870c0f881403fdd7fa5d966dab73740d99072013b0946 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, com.redhat.component=centos-stream-container, architecture=x86_64, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vendor=Red Hat, Inc., name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754) 2024-04-26T15:20:45.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:44 smithi112 podman[21667]: 2024-04-26 15:20:44.984101525 +0000 UTC m=+0.466237536 container cleanup 29e45d1fb1b5bf6083a870c0f881403fdd7fa5d966dab73740d99072013b0946 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, version=8, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, release=754, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, vcs-type=git, distribution-scope=public, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-26T15:20:45.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:44 smithi112 bash[21667]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112 2024-04-26T15:20:45.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 bash[21695]: Error: no container with name or ID "ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon.smithi112" found: no such container 2024-04-26T15:20:45.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 podman[21681]: 2024-04-26 15:20:45.168095483 +0000 UTC m=+0.506482565 container remove 29e45d1fb1b5bf6083a870c0f881403fdd7fa5d966dab73740d99072013b0946 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, name=centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, version=8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8) 2024-04-26T15:20:45.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi112.service: Deactivated successfully. 2024-04-26T15:20:45.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 systemd[1]: Stopped Ceph mon.smithi112 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:20:45.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 systemd[1]: Starting Ceph mon.smithi112 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:20:45.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 podman[21761]: 2024-04-26 15:20:45.402322135 +0000 UTC m=+0.111384809 container create 44d7d076db9f0c19aa3332b4f7af3752e558546afd63a4481b68b6d57c406aa5 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, name=centos-stream, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, distribution-scope=public, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-type=git) 2024-04-26T15:20:45.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 podman[21761]: 2024-04-26 15:20:45.309422745 +0000 UTC m=+0.018485442 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-26T15:20:45.707 INFO:teuthology.orchestra.run.smithi112.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-26T15:20:45.831 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 podman[21761]: 2024-04-26 15:20:45.617364943 +0000 UTC m=+0.326427628 container init 44d7d076db9f0c19aa3332b4f7af3752e558546afd63a4481b68b6d57c406aa5 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, name=centos-stream, com.redhat.component=centos-stream-container, vcs-type=git, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-26T15:20:45.831 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 podman[21761]: 2024-04-26 15:20:45.620075234 +0000 UTC m=+0.329137909 container start 44d7d076db9f0c19aa3332b4f7af3752e558546afd63a4481b68b6d57c406aa5 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, name=centos-stream, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, GIT_CLEAN=True, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.openshift.expose-services=, io.buildah.version=1.19.8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-26T15:20:45.831 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T15:20:45.831 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: pidfile_write: ignore empty --pid-file 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: load: jerasure load: lrc 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: RocksDB version: 6.15.5 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Compile date Apr 18 2022 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: DB SUMMARY 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: DB Session ID: JGUO7FHJOPETL68AKPU8 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: CURRENT file: CURRENT 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi112/store.db dir, Total Num: 1, files: 000008.sst 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi112/store.db: 000010.log size: 82741 ; 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.error_if_exists: 0 2024-04-26T15:20:45.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.create_if_missing: 0 2024-04-26T15:20:45.833 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.paranoid_checks: 1 2024-04-26T15:20:45.833 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T15:20:45.833 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.env: 0x55989d8c6860 2024-04-26T15:20:45.833 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.fs: Posix File System 2024-04-26T15:20:45.833 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.info_log: 0x55989f0e3fa0 2024-04-26T15:20:45.833 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T15:20:45.833 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.statistics: (nil) 2024-04-26T15:20:45.833 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.use_fsync: 0 2024-04-26T15:20:45.833 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_log_file_size: 0 2024-04-26T15:20:45.833 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T15:20:45.833 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T15:20:45.833 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.allow_fallocate: 1 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.use_direct_reads: 0 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.db_log_dir: 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi112/store.db 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T15:20:45.834 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T15:20:45.835 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T15:20:45.835 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T15:20:45.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T15:20:45.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.write_buffer_manager: 0x55989f1d42d0 2024-04-26T15:20:45.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T15:20:45.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T15:20:45.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T15:20:45.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T15:20:45.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T15:20:45.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T15:20:45.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T15:20:45.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.unordered_write: 0 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.row_cache: None 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.wal_filter: None 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.preserve_deletes: 0 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.two_write_queues: 0 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T15:20:45.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.atomic_flush: 0 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.log_readahead_size: 0 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_background_jobs: 2 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_background_compactions: -1 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_subcompactions: 1 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T15:20:45.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_open_files: -1 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_background_flushes: -1 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Compression algorithms supported: 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: kZSTD supported: 0 2024-04-26T15:20:45.839 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: kXpressCompression supported: 0 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: kLZ4Compression supported: 1 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: kBZip2Compression supported: 0 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: kZlibCompression supported: 1 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: kSnappyCompression supported: 1 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi112/store.db/MANIFEST-000009 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.merge_operator: 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_filter: None 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T15:20:45.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55989f0afd00) 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: cache_index_and_filter_blocks: 1 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: pin_top_level_index_and_filter: 1 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: index_type: 0 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: data_block_index_type: 0 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: index_shortening: 1 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: hash_index_allow_collision: 1 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: checksum: 1 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: no_block_cache: 0 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_cache: 0x55989f11a170 2024-04-26T15:20:45.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_cache_name: BinnedLRUCache 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_cache_options: 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: capacity : 536870912 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: num_shard_bits : 4 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: strict_capacity_limit : 0 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: high_pri_pool_ratio: 0.000 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_cache_compressed: (nil) 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: persistent_cache: (nil) 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_size: 4096 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_size_deviation: 10 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_restart_interval: 16 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: index_block_restart_interval: 1 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: metadata_block_size: 4096 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: partition_filters: 0 2024-04-26T15:20:45.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: use_delta_encoding: 1 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: whole_key_filtering: 1 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: verify_compression: 0 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: read_amp_bytes_per_bit: 0 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: format_version: 4 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: enable_index_compression: 1 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_align: 0 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compression: NoCompression 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.num_levels: 7 2024-04-26T15:20:45.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T15:20:45.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T15:20:45.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T15:20:45.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.table_properties_collectors: 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.inplace_update_support: 0 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.bloom_locality: 0 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.max_successive_merges: 0 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T15:20:45.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.ttl: 2592000 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.enable_blob_files: false 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.min_blob_size: 0 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi112/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-26T15:20:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714144845645792, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714144845646548, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79423, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77687, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71996, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714144845, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "74d6ffdc-59ef-41f9-a9fa-7b9b609f0dc8", "db_session_id": "JGUO7FHJOPETL68AKPU8"}} 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714144845647005, "job": 1, "event": "recovery_finished"} 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi112/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55989f100a80 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: DB pointer 0x55989f110000 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** DB Stats ** 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T15:20:45.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** Compaction Stats [default] ** 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: L0 2/0 79.36 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.1 0.00 0.00 1 0.001 0 0 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Sum 2/0 79.36 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.1 0.00 0.00 1 0.001 0 0 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.1 0.00 0.00 1 0.001 0 0 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** Compaction Stats [default] ** 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:20:45.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 145.1 0.00 0.00 1 0.001 0 0 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative compaction: 0.00 GB write, 24.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval compaction: 0.00 GB write, 24.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** Compaction Stats [default] ** 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:20:45.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: L0 2/0 79.36 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.1 0.00 0.00 1 0.001 0 0 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Sum 2/0 79.36 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.1 0.00 0.00 1 0.001 0 0 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** Compaction Stats [default] ** 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 145.1 0.00 0.00 1 0.001 0 0 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative compaction: 0.00 GB write, 24.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:20:45.852 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: starting mon.smithi112 rank 0 at public addrs [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] at bind addrs [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi112 fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: mon.smithi112@-1(???) e1 preinit fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: mon.smithi112@-1(???).mds e1 new map 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: mon.smithi112@-1(???).mds e1 print_map 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: e1 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: legacy client fscid: -1 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: No filesystems configured 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: mon.smithi112@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T15:20:45.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: mon.smithi112@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: mon.smithi112@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: mon.smithi112@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: mon.smithi112@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: expand_channel_meta expand map: {default=false} 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: expand_channel_meta from 'false' to 'false' 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: expand_channel_meta expanded map: {default=false} 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: expand_channel_meta expand map: {default=info} 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: expand_channel_meta from 'info' to 'info' 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: expand_channel_meta expanded map: {default=info} 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: expand_channel_meta expand map: {default=daemon} 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: expand_channel_meta from 'daemon' to 'daemon' 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: expand_channel_meta expanded map: {default=daemon} 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: expand_channel_meta expand map: {default=debug} 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: expand_channel_meta from 'debug' to 'debug' 2024-04-26T15:20:45.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: expand_channel_meta expanded map: {default=debug} 2024-04-26T15:20:45.855 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 bash[21761]: 44d7d076db9f0c19aa3332b4f7af3752e558546afd63a4481b68b6d57c406aa5 2024-04-26T15:20:45.855 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 systemd[1]: Started Ceph mon.smithi112 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:20:45.855 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: mon.smithi112 is new leader, mons smithi112 in quorum (ranks 0) 2024-04-26T15:20:45.855 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: monmap e1: 1 mons at {smithi112=[v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0]} 2024-04-26T15:20:45.855 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: fsmap 2024-04-26T15:20:45.855 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T15:20:45.855 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:45 smithi112 ceph-mon[21780]: mgrmap e1: no daemons active 2024-04-26T15:20:46.832 INFO:teuthology.orchestra.run.smithi112.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-26T15:20:46.833 INFO:teuthology.orchestra.run.smithi112.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-26T15:20:46.833 INFO:teuthology.orchestra.run.smithi112.stderr:Creating mgr... 2024-04-26T15:20:46.833 INFO:teuthology.orchestra.run.smithi112.stderr:Verifying port 9283 ... 2024-04-26T15:20:47.001 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: Failed to reset failed state of unit ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mgr.smithi112.llioug.service: Unit ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mgr.smithi112.llioug.service not loaded. 2024-04-26T15:20:47.008 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: Created symlink /etc/systemd/system/ceph-71e28ad8-03e0-11ef-bc93-c7b262605968.target.wants/ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mgr.smithi112.llioug.service → /etc/systemd/system/ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@.service. 2024-04-26T15:20:47.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:47 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1574368971' entity='client.admin' 2024-04-26T15:20:48.140 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: disabled 2024-04-26T15:20:48.148 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: inactive 2024-04-26T15:20:48.149 INFO:teuthology.orchestra.run.smithi112.stderr:firewalld.service is not enabled 2024-04-26T15:20:48.149 INFO:teuthology.orchestra.run.smithi112.stderr:Not possible to enable service . firewalld.service is not available 2024-04-26T15:20:48.159 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: disabled 2024-04-26T15:20:48.167 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: inactive 2024-04-26T15:20:48.168 INFO:teuthology.orchestra.run.smithi112.stderr:firewalld.service is not enabled 2024-04-26T15:20:48.168 INFO:teuthology.orchestra.run.smithi112.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-26T15:20:48.168 INFO:teuthology.orchestra.run.smithi112.stderr:Waiting for mgr to start... 2024-04-26T15:20:48.168 INFO:teuthology.orchestra.run.smithi112.stderr:Waiting for mgr... 2024-04-26T15:20:48.711 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:48.711 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: { 2024-04-26T15:20:48.711 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "fsid": "71e28ad8-03e0-11ef-bc93-c7b262605968", 2024-04-26T15:20:48.711 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "health": { 2024-04-26T15:20:48.711 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-26T15:20:48.711 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "checks": {}, 2024-04-26T15:20:48.711 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "mutes": [] 2024-04-26T15:20:48.711 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:48.711 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-26T15:20:48.711 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "quorum": [ 2024-04-26T15:20:48.711 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 0 2024-04-26T15:20:48.711 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ], 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "smithi112" 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ], 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "quorum_age": 3, 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "monmap": { 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "osdmap": { 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-26T15:20:48.712 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "pgmap": { 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "fsmap": { 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-26T15:20:48.713 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "available": false, 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "modules": [ 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "iostat", 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "nfs", 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "restful" 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ], 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "servicemap": { 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "modified": "2024-04-26T15:20:40.884404+0000", 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:48.714 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-26T15:20:48.715 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: } 2024-04-26T15:20:48.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:48 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2902764243' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T15:20:49.335 INFO:teuthology.orchestra.run.smithi112.stderr:mgr not available, waiting (1/15)... 2024-04-26T15:20:51.844 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:51.844 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: { 2024-04-26T15:20:51.844 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "fsid": "71e28ad8-03e0-11ef-bc93-c7b262605968", 2024-04-26T15:20:51.844 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "health": { 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "checks": {}, 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "mutes": [] 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "quorum": [ 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 0 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ], 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "smithi112" 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ], 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "monmap": { 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-26T15:20:51.845 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "osdmap": { 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "pgmap": { 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-26T15:20:51.846 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "fsmap": { 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "available": false, 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "modules": [ 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "iostat", 2024-04-26T15:20:51.847 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "nfs", 2024-04-26T15:20:51.848 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "restful" 2024-04-26T15:20:51.848 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ], 2024-04-26T15:20:51.848 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:20:51.848 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:51.848 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "servicemap": { 2024-04-26T15:20:51.848 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:51.848 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "modified": "2024-04-26T15:20:40.884404+0000", 2024-04-26T15:20:51.848 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:20:51.848 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:51.848 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-26T15:20:51.848 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: } 2024-04-26T15:20:51.951 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:51 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/838765181' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T15:20:52.475 INFO:teuthology.orchestra.run.smithi112.stderr:mgr not available, waiting (2/15)... 2024-04-26T15:20:54.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: Activating manager daemon smithi112.llioug 2024-04-26T15:20:54.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: mgrmap e2: smithi112.llioug(active, starting, since 0.00264943s) 2024-04-26T15:20:54.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: from='mgr.14100 172.21.15.112:0/2724953820' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:20:54.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: from='mgr.14100 172.21.15.112:0/2724953820' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:20:54.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: from='mgr.14100 172.21.15.112:0/2724953820' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:20:54.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: from='mgr.14100 172.21.15.112:0/2724953820' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:20:54.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: from='mgr.14100 172.21.15.112:0/2724953820' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi112.llioug", "id": "smithi112.llioug"}]: dispatch 2024-04-26T15:20:54.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: Manager daemon smithi112.llioug is now available 2024-04-26T15:20:54.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: from='mgr.14100 172.21.15.112:0/2724953820' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:20:54.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: from='mgr.14100 172.21.15.112:0/2724953820' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:20:54.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: from='mgr.14100 172.21.15.112:0/2724953820' entity='mgr.smithi112.llioug' 2024-04-26T15:20:54.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: from='mgr.14100 172.21.15.112:0/2724953820' entity='mgr.smithi112.llioug' 2024-04-26T15:20:54.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:54 smithi112 ceph-mon[21780]: from='mgr.14100 172.21.15.112:0/2724953820' entity='mgr.smithi112.llioug' 2024-04-26T15:20:54.970 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:54.970 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: { 2024-04-26T15:20:54.970 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "fsid": "71e28ad8-03e0-11ef-bc93-c7b262605968", 2024-04-26T15:20:54.970 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "health": { 2024-04-26T15:20:54.970 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-26T15:20:54.970 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "checks": {}, 2024-04-26T15:20:54.970 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "mutes": [] 2024-04-26T15:20:54.970 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:54.970 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-26T15:20:54.970 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "quorum": [ 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 0 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ], 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "smithi112" 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ], 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "monmap": { 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "osdmap": { 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-26T15:20:54.971 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-26T15:20:54.972 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-26T15:20:54.975 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-26T15:20:54.975 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-26T15:20:54.975 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:54.975 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "pgmap": { 2024-04-26T15:20:54.975 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-26T15:20:54.975 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-26T15:20:54.975 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-26T15:20:54.975 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-26T15:20:54.975 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-26T15:20:54.975 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-26T15:20:54.975 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-26T15:20:54.976 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-26T15:20:54.976 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:54.976 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "fsmap": { 2024-04-26T15:20:54.976 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:54.976 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-26T15:20:54.976 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-26T15:20:54.976 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:54.976 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-26T15:20:54.976 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "available": false, 2024-04-26T15:20:54.976 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-26T15:20:54.976 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "modules": [ 2024-04-26T15:20:54.976 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "iostat", 2024-04-26T15:20:54.977 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "nfs", 2024-04-26T15:20:54.977 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "restful" 2024-04-26T15:20:54.977 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ], 2024-04-26T15:20:54.977 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:20:54.977 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:54.977 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "servicemap": { 2024-04-26T15:20:54.977 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:54.977 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "modified": "2024-04-26T15:20:40.884404+0000", 2024-04-26T15:20:54.977 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:20:54.977 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:54.977 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-26T15:20:54.977 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: } 2024-04-26T15:20:55.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:55 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1765958190' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T15:20:55.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:55 smithi112 ceph-mon[21780]: mgrmap e3: smithi112.llioug(active, since 1.00525s) 2024-04-26T15:20:55.564 INFO:teuthology.orchestra.run.smithi112.stderr:mgr not available, waiting (3/15)... 2024-04-26T15:20:57.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:57 smithi112 ceph-mon[21780]: mgrmap e4: smithi112.llioug(active, since 2s) 2024-04-26T15:20:58.239 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:58.239 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: { 2024-04-26T15:20:58.239 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "fsid": "71e28ad8-03e0-11ef-bc93-c7b262605968", 2024-04-26T15:20:58.239 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "health": { 2024-04-26T15:20:58.239 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-26T15:20:58.239 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "checks": {}, 2024-04-26T15:20:58.239 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "mutes": [] 2024-04-26T15:20:58.239 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:58.239 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "quorum": [ 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 0 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ], 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "smithi112" 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ], 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "monmap": { 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "osdmap": { 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:58.240 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-26T15:20:58.241 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-26T15:20:58.241 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-26T15:20:58.241 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-26T15:20:58.244 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-26T15:20:58.244 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-26T15:20:58.244 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:58.244 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "pgmap": { 2024-04-26T15:20:58.244 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-26T15:20:58.244 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-26T15:20:58.244 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-26T15:20:58.244 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-26T15:20:58.244 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-26T15:20:58.244 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-26T15:20:58.244 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-26T15:20:58.244 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "fsmap": { 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "available": true, 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "modules": [ 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "iostat", 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "nfs", 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "restful" 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ], 2024-04-26T15:20:58.245 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:20:58.246 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:58.246 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "servicemap": { 2024-04-26T15:20:58.246 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:20:58.246 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "modified": "2024-04-26T15:20:40.884404+0000", 2024-04-26T15:20:58.246 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:20:58.246 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: }, 2024-04-26T15:20:58.246 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-26T15:20:58.246 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: } 2024-04-26T15:20:58.804 INFO:teuthology.orchestra.run.smithi112.stderr:mgr is available 2024-04-26T15:20:58.928 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:58 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/400891734' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T15:20:59.442 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: [global] 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: fsid = 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: [mgr] 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: [osd] 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-26T15:20:59.443 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-26T15:20:59.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:20:59 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/4154699886' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T15:20:59.988 INFO:teuthology.orchestra.run.smithi112.stderr:Enabling cephadm module... 2024-04-26T15:21:00.940 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:00 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/133103272' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-26T15:21:01.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:01 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/133103272' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-26T15:21:01.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:01 smithi112 ceph-mon[21780]: mgrmap e5: smithi112.llioug(active, since 6s) 2024-04-26T15:21:01.980 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: { 2024-04-26T15:21:01.980 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-26T15:21:01.980 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "available": true, 2024-04-26T15:21:01.980 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "active_name": "smithi112.llioug", 2024-04-26T15:21:01.980 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-26T15:21:01.980 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: } 2024-04-26T15:21:02.625 INFO:teuthology.orchestra.run.smithi112.stderr:Waiting for the mgr to restart... 2024-04-26T15:21:02.625 INFO:teuthology.orchestra.run.smithi112.stderr:Waiting for mgr epoch 5... 2024-04-26T15:21:02.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:02 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/3840259454' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T15:21:06.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:06 smithi112 ceph-mon[21780]: Active manager daemon smithi112.llioug restarted 2024-04-26T15:21:06.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:06 smithi112 ceph-mon[21780]: Activating manager daemon smithi112.llioug 2024-04-26T15:21:06.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:06 smithi112 ceph-mon[21780]: osdmap e2: 0 total, 0 up, 0 in 2024-04-26T15:21:07.669 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: { 2024-04-26T15:21:07.669 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-26T15:21:07.669 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "initialized": true 2024-04-26T15:21:07.670 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: } 2024-04-26T15:21:07.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: mgrmap e6: smithi112.llioug(active, starting, since 0.00963506s) 2024-04-26T15:21:07.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:21:07.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi112.llioug", "id": "smithi112.llioug"}]: dispatch 2024-04-26T15:21:07.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:21:07.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:21:07.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:21:07.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: Manager daemon smithi112.llioug is now available 2024-04-26T15:21:07.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:07.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:07.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:21:07.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:21:07.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:21:07.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:21:07.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:07 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:21:08.489 INFO:teuthology.orchestra.run.smithi112.stderr:mgr epoch 5 is available 2024-04-26T15:21:08.489 INFO:teuthology.orchestra.run.smithi112.stderr:Setting orchestrator backend to cephadm... 2024-04-26T15:21:08.813 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:08 smithi112 ceph-mon[21780]: [26/Apr/2024:15:21:07] ENGINE Bus STARTING 2024-04-26T15:21:08.813 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:08 smithi112 ceph-mon[21780]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-26T15:21:08.813 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:08 smithi112 ceph-mon[21780]: mgrmap e7: smithi112.llioug(active, since 1.01487s) 2024-04-26T15:21:08.813 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:08 smithi112 ceph-mon[21780]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-26T15:21:08.814 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:08 smithi112 ceph-mon[21780]: [26/Apr/2024:15:21:07] ENGINE Serving on https://172.21.15.112:7150 2024-04-26T15:21:08.814 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:08 smithi112 ceph-mon[21780]: [26/Apr/2024:15:21:07] ENGINE Bus STARTED 2024-04-26T15:21:08.814 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:08 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:08.814 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:08 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:21:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:09 smithi112 ceph-mon[21780]: mgrmap e8: smithi112.llioug(active, since 2s) 2024-04-26T15:21:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:09 smithi112 ceph-mon[21780]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:09 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:09 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:21:10.356 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: value unchanged 2024-04-26T15:21:10.909 INFO:teuthology.orchestra.run.smithi112.stderr:Generating ssh key... 2024-04-26T15:21:11.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:11 smithi112 ceph-mon[21780]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:12.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:12 smithi112 ceph-mon[21780]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:12.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:12 smithi112 ceph-mon[21780]: Generating ssh key... 2024-04-26T15:21:12.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:12 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:12.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:12 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:12.844 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKAjNlPTlZYnNy4XVIqiYIQoOb63x+RM9ZgqxSAyOTL1tcoRX06I0fZ4kcr1qd/XjZL2uXYKAngT+8eVg+Xh4UOldOw+QgJqIr8A76BrcOUqIRlJtShJdhPItmhe7VkAb+3sloFRX1YRr7KzEKIsMaXwLf7jKZ8lqTNEvnpMv3U77BbthzUyulsrMbDu8QwdS+uojbaw71PL6WQluyQP66PW+9K7merDqdPARjz7hD+ZEXTqmKehueJYwAuoNig55OmnGO39yU0bjnxtuWxRZc2aQTE81tNqzCb8zfqjRWIssacqH/x9DUQRj2bx6YZh0l8qO20cVsNrOvPrwLy0n8uLp7H1uVVGLu6JJO/FTpouuu6N9qycqj6Be0zJO+CYUwiMbwy1bOl9MEksuxDYlaSfbr7Yd3Sqru5/TV9ljRSwDee/ZQrV+9/hMUG+qnJSSU76LwE97d0TeHNE3M0NFjPb8dwIji1wmP2f2whq/U94/g88OnTlXXpwcZtUs1V0c= ceph-71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:21:13.430 INFO:teuthology.orchestra.run.smithi112.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-26T15:21:13.430 INFO:teuthology.orchestra.run.smithi112.stderr:Adding key to root@localhost authorized_keys... 2024-04-26T15:21:13.430 INFO:teuthology.orchestra.run.smithi112.stderr:Adding host smithi112... 2024-04-26T15:21:14.025 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:13 smithi112 ceph-mon[21780]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:14.831 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: Added host 'smithi112' with addr '172.21.15.112' 2024-04-26T15:21:14.874 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:14 smithi112 ceph-mon[21780]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi112", "addr": "172.21.15.112", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:14.874 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:14 smithi112 ceph-mon[21780]: Deploying cephadm binary to smithi112 2024-04-26T15:21:15.438 INFO:teuthology.orchestra.run.smithi112.stderr:Deploying mon service with default placement... 2024-04-26T15:21:16.042 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:15 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:16.043 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:15 smithi112 ceph-mon[21780]: Added host smithi112 2024-04-26T15:21:16.043 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:15 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:21:16.043 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-26T15:21:16.788 INFO:teuthology.orchestra.run.smithi112.stderr:Deploying mgr service with default placement... 2024-04-26T15:21:17.393 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:17 smithi112 ceph-mon[21780]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:17.393 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:17 smithi112 ceph-mon[21780]: Saving service mon spec with placement count:5 2024-04-26T15:21:17.393 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:17 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:17.393 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-26T15:21:18.098 INFO:teuthology.orchestra.run.smithi112.stderr:Deploying crash service with default placement... 2024-04-26T15:21:18.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:18 smithi112 ceph-mon[21780]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:18.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:18 smithi112 ceph-mon[21780]: Saving service mgr spec with placement count:2 2024-04-26T15:21:18.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:18 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:18.690 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: Scheduled crash update... 2024-04-26T15:21:19.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:19 smithi112 ceph-mon[21780]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:19.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:19 smithi112 ceph-mon[21780]: Saving service crash spec with placement * 2024-04-26T15:21:19.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:19 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:19.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:19 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:19.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:19 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:19.858 INFO:teuthology.orchestra.run.smithi112.stderr:Deploying prometheus service with default placement... 2024-04-26T15:21:20.471 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-04-26T15:21:21.708 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:21 smithi112 ceph-mon[21780]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:21.708 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:21 smithi112 ceph-mon[21780]: Saving service prometheus spec with placement count:1 2024-04-26T15:21:21.708 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:21 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:21.708 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:21 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:21.710 INFO:teuthology.orchestra.run.smithi112.stderr:Deploying grafana service with default placement... 2024-04-26T15:21:22.365 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-04-26T15:21:22.932 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:22 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:22.932 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:22 smithi112 ceph-mon[21780]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:22.932 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:22 smithi112 ceph-mon[21780]: Saving service grafana spec with placement count:1 2024-04-26T15:21:22.932 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:22 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:22.934 INFO:teuthology.orchestra.run.smithi112.stderr:Deploying node-exporter service with default placement... 2024-04-26T15:21:23.962 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-04-26T15:21:24.855 INFO:teuthology.orchestra.run.smithi112.stderr:Deploying alertmanager service with default placement... 2024-04-26T15:21:25.105 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:24 smithi112 ceph-mon[21780]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:25.105 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:24 smithi112 ceph-mon[21780]: Saving service node-exporter spec with placement * 2024-04-26T15:21:25.105 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:24 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:25.499 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-04-26T15:21:26.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:26 smithi112 ceph-mon[21780]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:26.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:26 smithi112 ceph-mon[21780]: Saving service alertmanager spec with placement count:1 2024-04-26T15:21:26.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:26 smithi112 ceph-mon[21780]: from='mgr.14120 172.21.15.112:0/2071372580' entity='mgr.smithi112.llioug' 2024-04-26T15:21:27.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:27 smithi112 ceph-mon[21780]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:21:27.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:27 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/323639611' entity='client.admin' 2024-04-26T15:21:28.656 INFO:teuthology.orchestra.run.smithi112.stderr:Enabling the dashboard module... 2024-04-26T15:21:29.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:29 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1767984726' entity='client.admin' 2024-04-26T15:21:29.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:29 smithi112 ceph-mon[21780]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:21:30.323 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:30 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1035913048' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-26T15:21:31.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:31 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1035913048' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-26T15:21:31.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:31 smithi112 ceph-mon[21780]: mgrmap e9: smithi112.llioug(active, since 23s) 2024-04-26T15:21:31.339 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: { 2024-04-26T15:21:31.339 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-26T15:21:31.339 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "available": true, 2024-04-26T15:21:31.339 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "active_name": "smithi112.llioug", 2024-04-26T15:21:31.339 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-26T15:21:31.340 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: } 2024-04-26T15:21:32.003 INFO:teuthology.orchestra.run.smithi112.stderr:Waiting for the mgr to restart... 2024-04-26T15:21:32.003 INFO:teuthology.orchestra.run.smithi112.stderr:Waiting for mgr epoch 9... 2024-04-26T15:21:32.205 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:32 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1639828680' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T15:21:36.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:36 smithi112 ceph-mon[21780]: Active manager daemon smithi112.llioug restarted 2024-04-26T15:21:36.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:36 smithi112 ceph-mon[21780]: Activating manager daemon smithi112.llioug 2024-04-26T15:21:36.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:36 smithi112 ceph-mon[21780]: osdmap e3: 0 total, 0 up, 0 in 2024-04-26T15:21:37.671 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: { 2024-04-26T15:21:37.672 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-26T15:21:37.672 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: "initialized": true 2024-04-26T15:21:37.672 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: } 2024-04-26T15:21:37.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:37 smithi112 ceph-mon[21780]: mgrmap e10: smithi112.llioug(active, starting, since 0.567755s) 2024-04-26T15:21:37.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:37 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:21:37.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:37 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi112.llioug", "id": "smithi112.llioug"}]: dispatch 2024-04-26T15:21:37.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:37 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:21:37.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:37 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:21:37.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:37 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:21:37.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:37 smithi112 ceph-mon[21780]: Manager daemon smithi112.llioug is now available 2024-04-26T15:21:37.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:37 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:21:37.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:37 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:21:37.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:37 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:21:37.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:37 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:21:37.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:37 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:21:38.330 INFO:teuthology.orchestra.run.smithi112.stderr:mgr epoch 9 is available 2024-04-26T15:21:38.330 INFO:teuthology.orchestra.run.smithi112.stderr:Generating a dashboard self-signed certificate... 2024-04-26T15:21:38.936 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:38 smithi112 ceph-mon[21780]: [26/Apr/2024:15:21:37] ENGINE Bus STARTING 2024-04-26T15:21:38.936 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:38 smithi112 ceph-mon[21780]: [26/Apr/2024:15:21:37] ENGINE Serving on https://172.21.15.112:7150 2024-04-26T15:21:38.936 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:38 smithi112 ceph-mon[21780]: [26/Apr/2024:15:21:37] ENGINE Bus STARTED 2024-04-26T15:21:38.936 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:38 smithi112 ceph-mon[21780]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-26T15:21:38.936 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:38 smithi112 ceph-mon[21780]: mgrmap e11: smithi112.llioug(active, since 1.57011s) 2024-04-26T15:21:38.936 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:38 smithi112 ceph-mon[21780]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-26T15:21:39.127 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-26T15:21:39.673 INFO:teuthology.orchestra.run.smithi112.stderr:Creating initial admin user... 2024-04-26T15:21:39.790 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:39 smithi112 ceph-mon[21780]: mgrmap e12: smithi112.llioug(active, since 2s) 2024-04-26T15:21:39.790 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:39 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:21:39.790 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:39 smithi112 ceph-mon[21780]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:39.791 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:39 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:21:39.791 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:39 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:21:40.639 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$KOGyfpbBEAApvviiFpgtqOYer.sGZE5N3bOGfehGRDzvjl21vyf9u", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1714144900, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-26T15:21:41.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:41 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:21:41.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:41 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:21:42.072 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:42 smithi112 ceph-mon[21780]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:42.074 INFO:teuthology.orchestra.run.smithi112.stderr:Fetching dashboard port number... 2024-04-26T15:21:42.697 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: 8443 2024-04-26T15:21:43.327 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:43 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/4100413318' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-26T15:21:43.338 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: disabled 2024-04-26T15:21:43.344 INFO:teuthology.orchestra.run.smithi112.stderr:systemctl: inactive 2024-04-26T15:21:43.345 INFO:teuthology.orchestra.run.smithi112.stderr:firewalld.service is not enabled 2024-04-26T15:21:43.345 INFO:teuthology.orchestra.run.smithi112.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-26T15:21:43.347 INFO:teuthology.orchestra.run.smithi112.stderr:Ceph Dashboard is now available at: 2024-04-26T15:21:43.347 INFO:teuthology.orchestra.run.smithi112.stderr: 2024-04-26T15:21:43.347 INFO:teuthology.orchestra.run.smithi112.stderr: URL: https://smithi112:8443/ 2024-04-26T15:21:43.347 INFO:teuthology.orchestra.run.smithi112.stderr: User: admin 2024-04-26T15:21:43.347 INFO:teuthology.orchestra.run.smithi112.stderr: Password: fh0ww32vjz 2024-04-26T15:21:43.347 INFO:teuthology.orchestra.run.smithi112.stderr: 2024-04-26T15:21:43.347 INFO:teuthology.orchestra.run.smithi112.stderr:Enabling autotune for osd_memory_target 2024-04-26T15:21:44.258 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:44 smithi112 ceph-mon[21780]: mgrmap e13: smithi112.llioug(active, since 6s) 2024-04-26T15:21:45.252 INFO:teuthology.orchestra.run.smithi112.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-26T15:21:45.894 INFO:teuthology.orchestra.run.smithi112.stderr:You can access the Ceph CLI with: 2024-04-26T15:21:45.895 INFO:teuthology.orchestra.run.smithi112.stderr: 2024-04-26T15:21:45.895 INFO:teuthology.orchestra.run.smithi112.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-26T15:21:45.895 INFO:teuthology.orchestra.run.smithi112.stderr: 2024-04-26T15:21:45.895 INFO:teuthology.orchestra.run.smithi112.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-26T15:21:45.895 INFO:teuthology.orchestra.run.smithi112.stderr: 2024-04-26T15:21:45.895 INFO:teuthology.orchestra.run.smithi112.stderr: ceph telemetry on 2024-04-26T15:21:45.895 INFO:teuthology.orchestra.run.smithi112.stderr: 2024-04-26T15:21:45.895 INFO:teuthology.orchestra.run.smithi112.stderr:For more information see: 2024-04-26T15:21:45.895 INFO:teuthology.orchestra.run.smithi112.stderr: 2024-04-26T15:21:45.895 INFO:teuthology.orchestra.run.smithi112.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-26T15:21:45.895 INFO:teuthology.orchestra.run.smithi112.stderr: 2024-04-26T15:21:45.895 INFO:teuthology.orchestra.run.smithi112.stderr:Bootstrap complete. 2024-04-26T15:21:45.928 INFO:tasks.cephadm:Fetching config... 2024-04-26T15:21:45.928 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-26T15:21:45.928 DEBUG:teuthology.orchestra.run.smithi112:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-26T15:21:45.944 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-26T15:21:45.944 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-26T15:21:45.944 DEBUG:teuthology.orchestra.run.smithi112:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-26T15:21:45.998 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-26T15:21:45.998 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-26T15:21:45.998 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/keyring of=/dev/stdout 2024-04-26T15:21:46.062 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-26T15:21:46.062 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-26T15:21:46.062 DEBUG:teuthology.orchestra.run.smithi112:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-26T15:21:46.117 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-26T15:21:46.117 DEBUG:teuthology.orchestra.run.smithi112:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKAjNlPTlZYnNy4XVIqiYIQoOb63x+RM9ZgqxSAyOTL1tcoRX06I0fZ4kcr1qd/XjZL2uXYKAngT+8eVg+Xh4UOldOw+QgJqIr8A76BrcOUqIRlJtShJdhPItmhe7VkAb+3sloFRX1YRr7KzEKIsMaXwLf7jKZ8lqTNEvnpMv3U77BbthzUyulsrMbDu8QwdS+uojbaw71PL6WQluyQP66PW+9K7merDqdPARjz7hD+ZEXTqmKehueJYwAuoNig55OmnGO39yU0bjnxtuWxRZc2aQTE81tNqzCb8zfqjRWIssacqH/x9DUQRj2bx6YZh0l8qO20cVsNrOvPrwLy0n8uLp7H1uVVGLu6JJO/FTpouuu6N9qycqj6Be0zJO+CYUwiMbwy1bOl9MEksuxDYlaSfbr7Yd3Sqru5/TV9ljRSwDee/ZQrV+9/hMUG+qnJSSU76LwE97d0TeHNE3M0NFjPb8dwIji1wmP2f2whq/U94/g88OnTlXXpwcZtUs1V0c= ceph-71e28ad8-03e0-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T15:21:46.188 INFO:teuthology.orchestra.run.smithi112.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKAjNlPTlZYnNy4XVIqiYIQoOb63x+RM9ZgqxSAyOTL1tcoRX06I0fZ4kcr1qd/XjZL2uXYKAngT+8eVg+Xh4UOldOw+QgJqIr8A76BrcOUqIRlJtShJdhPItmhe7VkAb+3sloFRX1YRr7KzEKIsMaXwLf7jKZ8lqTNEvnpMv3U77BbthzUyulsrMbDu8QwdS+uojbaw71PL6WQluyQP66PW+9K7merDqdPARjz7hD+ZEXTqmKehueJYwAuoNig55OmnGO39yU0bjnxtuWxRZc2aQTE81tNqzCb8zfqjRWIssacqH/x9DUQRj2bx6YZh0l8qO20cVsNrOvPrwLy0n8uLp7H1uVVGLu6JJO/FTpouuu6N9qycqj6Be0zJO+CYUwiMbwy1bOl9MEksuxDYlaSfbr7Yd3Sqru5/TV9ljRSwDee/ZQrV+9/hMUG+qnJSSU76LwE97d0TeHNE3M0NFjPb8dwIji1wmP2f2whq/U94/g88OnTlXXpwcZtUs1V0c= ceph-71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:21:46.198 DEBUG:teuthology.orchestra.run.smithi164:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKAjNlPTlZYnNy4XVIqiYIQoOb63x+RM9ZgqxSAyOTL1tcoRX06I0fZ4kcr1qd/XjZL2uXYKAngT+8eVg+Xh4UOldOw+QgJqIr8A76BrcOUqIRlJtShJdhPItmhe7VkAb+3sloFRX1YRr7KzEKIsMaXwLf7jKZ8lqTNEvnpMv3U77BbthzUyulsrMbDu8QwdS+uojbaw71PL6WQluyQP66PW+9K7merDqdPARjz7hD+ZEXTqmKehueJYwAuoNig55OmnGO39yU0bjnxtuWxRZc2aQTE81tNqzCb8zfqjRWIssacqH/x9DUQRj2bx6YZh0l8qO20cVsNrOvPrwLy0n8uLp7H1uVVGLu6JJO/FTpouuu6N9qycqj6Be0zJO+CYUwiMbwy1bOl9MEksuxDYlaSfbr7Yd3Sqru5/TV9ljRSwDee/ZQrV+9/hMUG+qnJSSU76LwE97d0TeHNE3M0NFjPb8dwIji1wmP2f2whq/U94/g88OnTlXXpwcZtUs1V0c= ceph-71e28ad8-03e0-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T15:21:46.239 INFO:teuthology.orchestra.run.smithi164.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKAjNlPTlZYnNy4XVIqiYIQoOb63x+RM9ZgqxSAyOTL1tcoRX06I0fZ4kcr1qd/XjZL2uXYKAngT+8eVg+Xh4UOldOw+QgJqIr8A76BrcOUqIRlJtShJdhPItmhe7VkAb+3sloFRX1YRr7KzEKIsMaXwLf7jKZ8lqTNEvnpMv3U77BbthzUyulsrMbDu8QwdS+uojbaw71PL6WQluyQP66PW+9K7merDqdPARjz7hD+ZEXTqmKehueJYwAuoNig55OmnGO39yU0bjnxtuWxRZc2aQTE81tNqzCb8zfqjRWIssacqH/x9DUQRj2bx6YZh0l8qO20cVsNrOvPrwLy0n8uLp7H1uVVGLu6JJO/FTpouuu6N9qycqj6Be0zJO+CYUwiMbwy1bOl9MEksuxDYlaSfbr7Yd3Sqru5/TV9ljRSwDee/ZQrV+9/hMUG+qnJSSU76LwE97d0TeHNE3M0NFjPb8dwIji1wmP2f2whq/U94/g88OnTlXXpwcZtUs1V0c= ceph-71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:21:46.250 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-26T15:21:46.357 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:46 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2581341506' entity='client.admin' 2024-04-26T15:21:50.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:50 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:21:50.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:50 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd/host:smithi112", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:21:50.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:50 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:21:50.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:50 smithi112 ceph-mon[21780]: Deploying daemon alertmanager.smithi112 on smithi112 2024-04-26T15:21:50.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:50 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2425200428' entity='client.admin' 2024-04-26T15:21:50.676 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-26T15:21:50.676 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-26T15:21:54.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:54 smithi112 ceph-mon[21780]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:21:54.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:54 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:21:54.531 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi164 2024-04-26T15:21:54.532 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-04-26T15:21:54.532 DEBUG:teuthology.orchestra.run.smithi164:> dd of=/etc/ceph/ceph.conf 2024-04-26T15:21:54.550 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-04-26T15:21:54.550 DEBUG:teuthology.orchestra.run.smithi164:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:21:54.608 INFO:tasks.cephadm:Adding host smithi164 to orchestrator... 2024-04-26T15:21:54.608 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch host add smithi164 2024-04-26T15:21:57.854 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:57 smithi112 ceph-mon[21780]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:21:59.737 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:59 smithi112 ceph-mon[21780]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:21:59.737 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:59 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:21:59.737 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:59 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:21:59.737 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:59 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-26T15:21:59.737 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:21:59 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:21:59.975 INFO:teuthology.orchestra.run.smithi112.stdout:Added host 'smithi164' with addr '172.21.15.164' 2024-04-26T15:22:00.611 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-26T15:22:01.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:00 smithi112 ceph-mon[21780]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi164", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:22:01.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:00 smithi112 ceph-mon[21780]: Deploying cephadm binary to smithi164 2024-04-26T15:22:01.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:00 smithi112 ceph-mon[21780]: Deploying daemon crash.smithi112 on smithi112 2024-04-26T15:22:01.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:00 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:22:01.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:00 smithi112 ceph-mon[21780]: Added host smithi164 2024-04-26T15:22:02.138 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:01 smithi112 ceph-mon[21780]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:02.138 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:01 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:22:03.484 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:22:03.484 INFO:teuthology.orchestra.run.smithi112.stdout:[{"addr": "172.21.15.112", "hostname": "smithi112", "labels": [], "status": ""}, {"addr": "172.21.15.164", "hostname": "smithi164", "labels": [], "status": ""}] 2024-04-26T15:22:04.225 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:03 smithi112 ceph-mon[21780]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:04.868 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-26T15:22:04.868 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-26T15:22:05.183 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:04 smithi112 ceph-mon[21780]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:22:05.183 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:04 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:22:05.184 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:04 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:22:05.184 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:04 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:22:05.184 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:04 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T15:22:05.184 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:04 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:22:06.199 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:05 smithi112 ceph-mon[21780]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T15:22:06.199 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:05 smithi112 ceph-mon[21780]: Deploying daemon grafana.smithi112 on smithi112 2024-04-26T15:22:06.199 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:05 smithi112 ceph-mon[21780]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:07.708 INFO:teuthology.orchestra.run.smithi112.stderr:adjusted tunables profile to default 2024-04-26T15:22:07.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:07 smithi112 ceph-mon[21780]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:07.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:07 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:22:07.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:07 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2405343121' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-26T15:22:08.431 INFO:tasks.cephadm:Adding mon.smithi112 on smithi112 2024-04-26T15:22:08.431 INFO:tasks.cephadm:Adding mon.smithi164 on smithi164 2024-04-26T15:22:08.432 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch apply mon '2;smithi112:172.21.15.112=smithi112;smithi164:172.21.15.164=smithi164' 2024-04-26T15:22:08.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:08 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2405343121' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T15:22:08.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:08 smithi112 ceph-mon[21780]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T15:22:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:09 smithi112 ceph-mon[21780]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:10.219 INFO:teuthology.orchestra.run.smithi164.stdout:Scheduled mon update... 2024-04-26T15:22:10.797 DEBUG:teuthology.orchestra.run.smithi164:mon.smithi164> sudo journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi164.service 2024-04-26T15:22:10.801 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-26T15:22:10.801 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:22:11.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:11 smithi112 ceph-mon[21780]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi112:172.21.15.112=smithi112;smithi164:172.21.15.164=smithi164", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:22:11.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:11 smithi112 ceph-mon[21780]: Saving service mon spec with placement smithi112:172.21.15.112=smithi112;smithi164:172.21.15.164=smithi164;count:2 2024-04-26T15:22:11.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:11 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:22:11.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:11 smithi112 ceph-mon[21780]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:12.579 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:22:12.579 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","modified":"2024-04-26T15:20:36.122604Z","created":"2024-04-26T15:20:36.122604Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T15:22:12.581 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-04-26T15:22:12.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:12 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/128975710' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:13.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:13 smithi112 ceph-mon[21780]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:14.141 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-26T15:22:14.141 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:22:15.977 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:22:16.021 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","modified":"2024-04-26T15:20:36.122604Z","created":"2024-04-26T15:20:36.122604Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T15:22:16.021 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-04-26T15:22:16.171 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:15 smithi112 ceph-mon[21780]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:16.880 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:16 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/1764736611' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:17.551 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-26T15:22:17.552 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:22:17.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:17 smithi112 ceph-mon[21780]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:17.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:17 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:22:19.016 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:18 smithi112 ceph-mon[21780]: Deploying daemon node-exporter.smithi112 on smithi112 2024-04-26T15:22:19.398 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:22:19.398 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","modified":"2024-04-26T15:20:36.122604Z","created":"2024-04-26T15:20:36.122604Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T15:22:19.399 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-04-26T15:22:20.059 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:19 smithi112 ceph-mon[21780]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:20.059 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:19 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/3686097140' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:20.972 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-26T15:22:20.972 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:22:21.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:21 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' 2024-04-26T15:22:21.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:21 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T15:22:21.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:21 smithi112 ceph-mon[21780]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:22 smithi112 ceph-mon[21780]: from='mgr.14162 172.21.15.112:0/945892608' entity='mgr.smithi112.llioug' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T15:22:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:22 smithi112 ceph-mon[21780]: mgrmap e14: smithi112.llioug(active, since 45s) 2024-04-26T15:22:22.837 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:22:22.837 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","modified":"2024-04-26T15:20:36.122604Z","created":"2024-04-26T15:20:36.122604Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T15:22:22.839 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-04-26T15:22:23.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:23 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/1638828709' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:24.366 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-26T15:22:24.367 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:22:26.194 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:22:26.194 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","modified":"2024-04-26T15:20:36.122604Z","created":"2024-04-26T15:20:36.122604Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T15:22:26.196 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-04-26T15:22:26.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:26 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/592663109' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:27.725 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-26T15:22:27.726 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: Active manager daemon smithi112.llioug restarted 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: Activating manager daemon smithi112.llioug 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: osdmap e5: 0 total, 0 up, 0 in 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: mgrmap e15: smithi112.llioug(active, starting, since 0.516465s) 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi112.llioug", "id": "smithi112.llioug"}]: dispatch 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: Manager daemon smithi112.llioug is now available 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:22:28.424 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:22:28.425 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:28.425 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:22:28.425 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:22:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:29 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:29 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:29 smithi112 ceph-mon[21780]: mgrmap e16: smithi112.llioug(active, since 1.51871s) 2024-04-26T15:22:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:29 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:29.838 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:22:29.838 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","modified":"2024-04-26T15:20:36.122604Z","created":"2024-04-26T15:20:36.122604Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T15:22:29.840 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-04-26T15:22:30.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:30 smithi112 ceph-mon[21780]: [26/Apr/2024:15:22:28] ENGINE Bus STARTING 2024-04-26T15:22:30.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:30 smithi112 ceph-mon[21780]: [26/Apr/2024:15:22:28] ENGINE Serving on https://172.21.15.112:7150 2024-04-26T15:22:30.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:30 smithi112 ceph-mon[21780]: [26/Apr/2024:15:22:28] ENGINE Bus STARTED 2024-04-26T15:22:30.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:30 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:30.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:30 smithi112 ceph-mon[21780]: mgrmap e17: smithi112.llioug(active, since 2s) 2024-04-26T15:22:30.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:30 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/64834786' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:31.966 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-26T15:22:31.966 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:22:34.266 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:22:34.266 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","modified":"2024-04-26T15:20:36.122604Z","created":"2024-04-26T15:20:36.122604Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T15:22:34.268 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-04-26T15:22:34.375 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:34 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:34.375 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:34 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:34.375 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:34 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd/host:smithi164", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:22:35.376 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:35 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/2749876842' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:35.861 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-26T15:22:35.862 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:22:36.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:36.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:36.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd/host:smithi112", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:22:36.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:36 smithi112 ceph-mon[21780]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-26T15:22:36.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:36.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:36.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:22:36.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-26T15:22:36.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:22:37.766 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:22:37.766 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","modified":"2024-04-26T15:20:36.122604Z","created":"2024-04-26T15:20:36.122604Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T15:22:37.768 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-04-26T15:22:37.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:37 smithi112 ceph-mon[21780]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:22:37.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:37 smithi112 ceph-mon[21780]: Deploying daemon crash.smithi164 on smithi164 2024-04-26T15:22:38.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:38 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/2012930676' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:39.274 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-26T15:22:39.274 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:22:41.361 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:22:41.361 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","modified":"2024-04-26T15:20:36.122604Z","created":"2024-04-26T15:20:36.122604Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T15:22:41.362 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-04-26T15:22:41.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:41 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/2288621955' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:43.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:43.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.tuhwus", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:22:43.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.tuhwus", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T15:22:43.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:22:43.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:22:43.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:42 smithi112 ceph-mon[21780]: Deploying daemon mgr.smithi164.tuhwus on smithi164 2024-04-26T15:22:43.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:43.361 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-26T15:22:43.362 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:22:45.312 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:22:45.312 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","modified":"2024-04-26T15:20:36.122604Z","created":"2024-04-26T15:20:36.122604Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T15:22:45.313 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-04-26T15:22:45.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:45 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/1184199844' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:46.897 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-26T15:22:46.897 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:22:47.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:47 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:47.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:47 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:22:47.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:47 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:22:47.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:47 smithi112 ceph-mon[21780]: Deploying daemon mon.smithi164 on smithi164 2024-04-26T15:22:48.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:48 smithi112 ceph-mon[21780]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:48.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:48 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:50.609 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:22:50.609 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","modified":"2024-04-26T15:20:36.122604Z","created":"2024-04-26T15:20:36.122604Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T15:22:50.611 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-04-26T15:22:50.841 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:50 smithi164 systemd[1]: Started Ceph mon.smithi164 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:22:51.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:50 smithi112 ceph-mon[21780]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:51.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:50 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/4204144816' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:52.487 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-26T15:22:52.487 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:22:56.096 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:22:56.096 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: mon.smithi112 calling monitor election 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/3831720296' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/crt"}]: dispatch 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: mon.smithi164 calling monitor election 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: mon.smithi112 is new leader, mons smithi112,smithi164 in quorum (ranks 0,1) 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: monmap e2: 2 mons at {smithi112=[v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0],smithi164=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0]} 2024-04-26T15:22:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: fsmap 2024-04-26T15:22:56.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: osdmap e5: 0 total, 0 up, 0 in 2024-04-26T15:22:56.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: mgrmap e17: smithi112.llioug(active, since 28s) 2024-04-26T15:22:56.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: Standby manager daemon smithi164.tuhwus started 2024-04-26T15:22:56.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/3831720296' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:22:56.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: overall HEALTH_OK 2024-04-26T15:22:56.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/3831720296' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/key"}]: dispatch 2024-04-26T15:22:56.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:56.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:55 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/3831720296' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: mon.smithi112 calling monitor election 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/3831720296' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/crt"}]: dispatch 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: mon.smithi164 calling monitor election 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:56.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: mon.smithi112 is new leader, mons smithi112,smithi164 in quorum (ranks 0,1) 2024-04-26T15:22:56.213 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: monmap e2: 2 mons at {smithi112=[v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0],smithi164=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0]} 2024-04-26T15:22:56.213 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: fsmap 2024-04-26T15:22:56.213 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: osdmap e5: 0 total, 0 up, 0 in 2024-04-26T15:22:56.213 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: mgrmap e17: smithi112.llioug(active, since 28s) 2024-04-26T15:22:56.213 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: Standby manager daemon smithi164.tuhwus started 2024-04-26T15:22:56.213 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/3831720296' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:22:56.213 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: overall HEALTH_OK 2024-04-26T15:22:56.213 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/3831720296' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/key"}]: dispatch 2024-04-26T15:22:56.213 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:56.213 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:55 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/3831720296' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:22:57.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:56 smithi112 ceph-mon[21780]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:57.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:56 smithi112 ceph-mon[21780]: Deploying daemon node-exporter.smithi164 on smithi164 2024-04-26T15:22:57.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:56 smithi112 ceph-mon[21780]: mgrmap e18: smithi112.llioug(active, since 28s), standbys: smithi164.tuhwus 2024-04-26T15:22:57.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:56 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi164.tuhwus", "id": "smithi164.tuhwus"}]: dispatch 2024-04-26T15:22:57.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:56 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:57.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:56 smithi164 ceph-mon[24045]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:57.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:56 smithi164 ceph-mon[24045]: Deploying daemon node-exporter.smithi164 on smithi164 2024-04-26T15:22:57.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:56 smithi164 ceph-mon[24045]: mgrmap e18: smithi112.llioug(active, since 28s), standbys: smithi164.tuhwus 2024-04-26T15:22:57.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:56 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi164.tuhwus", "id": "smithi164.tuhwus"}]: dispatch 2024-04-26T15:22:57.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:56 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:22:57.418 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-04-26T15:22:57.418 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":2,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","modified":"2024-04-26T15:22:50.791303Z","created":"2024-04-26T15:20:36.122604Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi164","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:3300","nonce":0},{"type":"v1","addr":"172.21.15.164:6789","nonce":0}]},"addr":"172.21.15.164:6789/0","public_addr":"172.21.15.164:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-26T15:22:57.420 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 2 2024-04-26T15:22:58.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:57 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/1585227166' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:58.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:57 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:58.261 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:57 smithi164 ceph-mon[24045]: from='client.? 172.21.15.164:0/1585227166' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:22:58.261 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:57 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:22:58.702 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-26T15:22:58.702 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-26T15:22:59.010 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:22:58 smithi112 ceph-mon[21780]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:22:59.068 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:22:58 smithi164 ceph-mon[24045]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:00.643 INFO:teuthology.orchestra.run.smithi112.stdout:# minimal ceph.conf for 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:23:00.643 INFO:teuthology.orchestra.run.smithi112.stdout:[global] 2024-04-26T15:23:00.643 INFO:teuthology.orchestra.run.smithi112.stdout: fsid = 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:23:00.643 INFO:teuthology.orchestra.run.smithi112.stdout: mon_host = [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] 2024-04-26T15:23:00.644 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:00.644 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:00 smithi112 ceph-mon[21780]: Deploying daemon prometheus.smithi112 on smithi112 2024-04-26T15:23:00.644 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:00 smithi112 ceph-mon[21780]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:00.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:00.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:00 smithi164 ceph-mon[24045]: Deploying daemon prometheus.smithi112 on smithi112 2024-04-26T15:23:00.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:00 smithi164 ceph-mon[24045]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:01.280 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-26T15:23:01.280 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-26T15:23:01.280 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T15:23:01.306 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-26T15:23:01.306 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:23:01.369 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-04-26T15:23:01.369 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T15:23:01.401 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-04-26T15:23:01.401 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:23:01.468 INFO:tasks.cephadm:Deploying OSDs... 2024-04-26T15:23:01.468 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-26T15:23:01.469 DEBUG:teuthology.orchestra.run.smithi112:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T15:23:01.483 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T15:23:01.483 DEBUG:teuthology.orchestra.run.smithi112:> stat /dev/vg_nvme/lv_1 2024-04-26T15:23:01.538 INFO:teuthology.orchestra.run.smithi112.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T15:23:01.538 INFO:teuthology.orchestra.run.smithi112.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:23:01.538 INFO:teuthology.orchestra.run.smithi112.stdout:Device: 5h/5d Inode: 664 Links: 1 2024-04-26T15:23:01.538 INFO:teuthology.orchestra.run.smithi112.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:23:01.538 INFO:teuthology.orchestra.run.smithi112.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:23:01.538 INFO:teuthology.orchestra.run.smithi112.stdout:Access: 2024-04-26 15:22:17.737825795 +0000 2024-04-26T15:23:01.538 INFO:teuthology.orchestra.run.smithi112.stdout:Modify: 2024-04-26 15:19:38.876178114 +0000 2024-04-26T15:23:01.538 INFO:teuthology.orchestra.run.smithi112.stdout:Change: 2024-04-26 15:19:38.876178114 +0000 2024-04-26T15:23:01.539 INFO:teuthology.orchestra.run.smithi112.stdout: Birth: 2024-04-26 15:19:38.876178114 +0000 2024-04-26T15:23:01.539 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T15:23:01.599 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:01 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/896882512' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:01.601 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records in 2024-04-26T15:23:01.601 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records out 2024-04-26T15:23:01.601 INFO:teuthology.orchestra.run.smithi112.stderr:512 bytes copied, 0.000197476 s, 2.6 MB/s 2024-04-26T15:23:01.602 DEBUG:teuthology.orchestra.run.smithi112:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T15:23:01.658 DEBUG:teuthology.orchestra.run.smithi112:> stat /dev/vg_nvme/lv_2 2024-04-26T15:23:01.713 INFO:teuthology.orchestra.run.smithi112.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T15:23:01.713 INFO:teuthology.orchestra.run.smithi112.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:23:01.713 INFO:teuthology.orchestra.run.smithi112.stdout:Device: 5h/5d Inode: 682 Links: 1 2024-04-26T15:23:01.713 INFO:teuthology.orchestra.run.smithi112.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:23:01.713 INFO:teuthology.orchestra.run.smithi112.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:23:01.714 INFO:teuthology.orchestra.run.smithi112.stdout:Access: 2024-04-26 15:22:17.737825795 +0000 2024-04-26T15:23:01.714 INFO:teuthology.orchestra.run.smithi112.stdout:Modify: 2024-04-26 15:19:39.214167697 +0000 2024-04-26T15:23:01.714 INFO:teuthology.orchestra.run.smithi112.stdout:Change: 2024-04-26 15:19:39.214167697 +0000 2024-04-26T15:23:01.714 INFO:teuthology.orchestra.run.smithi112.stdout: Birth: 2024-04-26 15:19:39.214167697 +0000 2024-04-26T15:23:01.714 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T15:23:01.776 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records in 2024-04-26T15:23:01.776 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records out 2024-04-26T15:23:01.776 INFO:teuthology.orchestra.run.smithi112.stderr:512 bytes copied, 0.000230817 s, 2.2 MB/s 2024-04-26T15:23:01.777 DEBUG:teuthology.orchestra.run.smithi112:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T15:23:01.834 DEBUG:teuthology.orchestra.run.smithi112:> stat /dev/vg_nvme/lv_3 2024-04-26T15:23:01.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:01 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/896882512' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:01.888 INFO:teuthology.orchestra.run.smithi112.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T15:23:01.888 INFO:teuthology.orchestra.run.smithi112.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:23:01.888 INFO:teuthology.orchestra.run.smithi112.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-26T15:23:01.888 INFO:teuthology.orchestra.run.smithi112.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:23:01.888 INFO:teuthology.orchestra.run.smithi112.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:23:01.888 INFO:teuthology.orchestra.run.smithi112.stdout:Access: 2024-04-26 15:22:17.738825768 +0000 2024-04-26T15:23:01.888 INFO:teuthology.orchestra.run.smithi112.stdout:Modify: 2024-04-26 15:19:39.555157188 +0000 2024-04-26T15:23:01.889 INFO:teuthology.orchestra.run.smithi112.stdout:Change: 2024-04-26 15:19:39.555157188 +0000 2024-04-26T15:23:01.889 INFO:teuthology.orchestra.run.smithi112.stdout: Birth: 2024-04-26 15:19:39.555157188 +0000 2024-04-26T15:23:01.889 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T15:23:01.950 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records in 2024-04-26T15:23:01.950 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records out 2024-04-26T15:23:01.950 INFO:teuthology.orchestra.run.smithi112.stderr:512 bytes copied, 0.000178149 s, 2.9 MB/s 2024-04-26T15:23:01.951 DEBUG:teuthology.orchestra.run.smithi112:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T15:23:02.006 DEBUG:teuthology.orchestra.run.smithi112:> stat /dev/vg_nvme/lv_4 2024-04-26T15:23:02.061 INFO:teuthology.orchestra.run.smithi112.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T15:23:02.105 INFO:teuthology.orchestra.run.smithi112.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:23:02.105 INFO:teuthology.orchestra.run.smithi112.stdout:Device: 5h/5d Inode: 720 Links: 1 2024-04-26T15:23:02.106 INFO:teuthology.orchestra.run.smithi112.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:23:02.106 INFO:teuthology.orchestra.run.smithi112.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:23:02.106 INFO:teuthology.orchestra.run.smithi112.stdout:Access: 2024-04-26 15:22:17.738825768 +0000 2024-04-26T15:23:02.106 INFO:teuthology.orchestra.run.smithi112.stdout:Modify: 2024-04-26 15:19:39.896146679 +0000 2024-04-26T15:23:02.106 INFO:teuthology.orchestra.run.smithi112.stdout:Change: 2024-04-26 15:19:39.896146679 +0000 2024-04-26T15:23:02.106 INFO:teuthology.orchestra.run.smithi112.stdout: Birth: 2024-04-26 15:19:39.896146679 +0000 2024-04-26T15:23:02.106 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T15:23:02.130 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records in 2024-04-26T15:23:02.130 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records out 2024-04-26T15:23:02.130 INFO:teuthology.orchestra.run.smithi112.stderr:512 bytes copied, 0.000217665 s, 2.4 MB/s 2024-04-26T15:23:02.131 DEBUG:teuthology.orchestra.run.smithi112:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T15:23:02.187 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-04-26T15:23:02.187 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T15:23:02.205 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T15:23:02.205 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_1 2024-04-26T15:23:02.263 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T15:23:02.263 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:23:02.263 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-26T15:23:02.263 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:23:02.264 INFO:teuthology.orchestra.run.smithi164.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:23:02.264 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2024-04-26 15:22:56.387445675 +0000 2024-04-26T15:23:02.264 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2024-04-26 15:19:38.875637343 +0000 2024-04-26T15:23:02.264 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2024-04-26 15:19:38.875637343 +0000 2024-04-26T15:23:02.264 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: 2024-04-26 15:19:38.875637343 +0000 2024-04-26T15:23:02.264 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T15:23:02.329 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2024-04-26T15:23:02.329 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2024-04-26T15:23:02.329 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000215173 s, 2.4 MB/s 2024-04-26T15:23:02.331 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T15:23:02.388 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_2 2024-04-26T15:23:02.445 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T15:23:02.445 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:23:02.445 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 682 Links: 1 2024-04-26T15:23:02.445 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:23:02.445 INFO:teuthology.orchestra.run.smithi164.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:23:02.445 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2024-04-26 15:22:56.387445675 +0000 2024-04-26T15:23:02.445 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2024-04-26 15:19:39.218631272 +0000 2024-04-26T15:23:02.445 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2024-04-26 15:19:39.218631272 +0000 2024-04-26T15:23:02.445 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: 2024-04-26 15:19:39.218631272 +0000 2024-04-26T15:23:02.446 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T15:23:02.509 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2024-04-26T15:23:02.510 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2024-04-26T15:23:02.510 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000251836 s, 2.0 MB/s 2024-04-26T15:23:02.511 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T15:23:02.569 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_3 2024-04-26T15:23:02.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:02 smithi164 ceph-mon[24045]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:02.612 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T15:23:02.612 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:23:02.612 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-04-26T15:23:02.613 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:23:02.613 INFO:teuthology.orchestra.run.smithi164.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:23:02.613 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2024-04-26 15:22:56.387445675 +0000 2024-04-26T15:23:02.613 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2024-04-26 15:19:39.555625308 +0000 2024-04-26T15:23:02.613 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2024-04-26 15:19:39.555625308 +0000 2024-04-26T15:23:02.613 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: 2024-04-26 15:19:39.555625308 +0000 2024-04-26T15:23:02.613 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T15:23:02.675 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2024-04-26T15:23:02.676 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2024-04-26T15:23:02.676 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000297901 s, 1.7 MB/s 2024-04-26T15:23:02.677 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T15:23:02.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:02 smithi112 ceph-mon[21780]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:02.733 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_4 2024-04-26T15:23:02.789 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T15:23:02.789 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:23:02.789 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 720 Links: 1 2024-04-26T15:23:02.789 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:23:02.789 INFO:teuthology.orchestra.run.smithi164.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:23:02.789 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2024-04-26 15:22:56.388445658 +0000 2024-04-26T15:23:02.789 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2024-04-26 15:19:39.888619414 +0000 2024-04-26T15:23:02.789 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2024-04-26 15:19:39.888619414 +0000 2024-04-26T15:23:02.789 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: 2024-04-26 15:19:39.888619414 +0000 2024-04-26T15:23:02.790 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T15:23:02.852 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2024-04-26T15:23:02.852 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2024-04-26T15:23:02.852 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000179223 s, 2.9 MB/s 2024-04-26T15:23:02.853 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T15:23:02.909 INFO:tasks.cephadm:Deploying osd.0 on smithi112 with /dev/vg_nvme/lv_4... 2024-04-26T15:23:02.909 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T15:23:03.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:04.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:04.991 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:04 smithi112 ceph-mon[21780]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:05.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:04 smithi164 ceph-mon[24045]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:05.829 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T15:23:05.829 INFO:teuthology.orchestra.run.smithi112.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T15:23:05.829 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 10+0 records in 2024-04-26T15:23:05.829 INFO:teuthology.orchestra.run.smithi112.stderr:10+0 records out 2024-04-26T15:23:05.830 INFO:teuthology.orchestra.run.smithi112.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0136709 s, 767 MB/s 2024-04-26T15:23:05.830 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping successful for: 2024-04-26T15:23:06.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:06 smithi112 ceph-mon[21780]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:07.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:06 smithi164 ceph-mon[24045]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:07.487 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi112:vg_nvme/lv_4 2024-04-26T15:23:08.876 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:08 smithi112 ceph-mon[21780]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:09.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:08 smithi164 ceph-mon[24045]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:10.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:10 smithi112 ceph-mon[21780]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:11.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:10 smithi164 ceph-mon[24045]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:12.938 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:12 smithi112 ceph-mon[21780]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:12.938 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:12 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:23:12.938 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:12 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:23:12.938 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:12 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:12 smithi164 ceph-mon[24045]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:12 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:23:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:12 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:23:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:12 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:14.345 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:14 smithi164 ceph-mon[24045]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:23:14.345 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:14.345 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:23:14.345 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:14.345 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:23:14.396 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:14 smithi112 ceph-mon[21780]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:23:14.396 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:14.397 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:23:14.397 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:14.397 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:23:15.117 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:15 smithi112 ceph-mon[21780]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:15.118 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:15 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:15.118 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:15 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2826347051' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57f44c85-dc23-45ed-909d-bfaa1b8bbbec"}]: dispatch 2024-04-26T15:23:15.118 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:15 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2826347051' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "57f44c85-dc23-45ed-909d-bfaa1b8bbbec"}]': finished 2024-04-26T15:23:15.118 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:15 smithi112 ceph-mon[21780]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T15:23:15.118 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:15 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:23:15.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:15 smithi164 ceph-mon[24045]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:15.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:15 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:15.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:15 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/2826347051' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57f44c85-dc23-45ed-909d-bfaa1b8bbbec"}]: dispatch 2024-04-26T15:23:15.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:15 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/2826347051' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "57f44c85-dc23-45ed-909d-bfaa1b8bbbec"}]': finished 2024-04-26T15:23:15.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:15 smithi164 ceph-mon[24045]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T15:23:15.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:15 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:23:16.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:16 smithi112 ceph-mon[21780]: Updating smithi164:/etc/ceph/ceph.conf 2024-04-26T15:23:16.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:16 smithi112 ceph-mon[21780]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:23:16.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:16 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:16.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:16 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/756586755' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:23:16.495 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:16 smithi164 ceph-mon[24045]: Updating smithi164:/etc/ceph/ceph.conf 2024-04-26T15:23:16.495 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:16 smithi164 ceph-mon[24045]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:23:16.495 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:16 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:16.496 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:16 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/756586755' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:23:17.402 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:17 smithi164 ceph-mon[24045]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:17.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:17 smithi112 ceph-mon[21780]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:18.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:18 smithi112 ceph-mon[21780]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:18.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:18 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:19.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:18 smithi164 ceph-mon[24045]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:19.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:18 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:19 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:19 smithi164 ceph-mon[24045]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-26T15:23:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:19 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:19 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:19 smithi164 ceph-mon[24045]: Reconfiguring mon.smithi112 (unknown last config time)... 2024-04-26T15:23:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:19 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:23:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:19 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:23:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:19 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:19 smithi164 ceph-mon[24045]: Reconfiguring daemon mon.smithi112 on smithi112 2024-04-26T15:23:20.198 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:19 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:20.198 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:19 smithi112 ceph-mon[21780]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-26T15:23:20.198 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:19 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:20.198 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:19 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:20.199 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:19 smithi112 ceph-mon[21780]: Reconfiguring mon.smithi112 (unknown last config time)... 2024-04-26T15:23:20.199 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:19 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:23:20.199 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:19 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:23:20.199 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:19 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:20.199 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:19 smithi112 ceph-mon[21780]: Reconfiguring daemon mon.smithi112 on smithi112 2024-04-26T15:23:21.284 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:21 smithi112 ceph-mon[21780]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:21.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:21 smithi164 ceph-mon[24045]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:22 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:22 smithi164 ceph-mon[24045]: Reconfiguring mgr.smithi112.llioug (unknown last config time)... 2024-04-26T15:23:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:22 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.llioug", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:23:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:22 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:23:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:22 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:22 smithi164 ceph-mon[24045]: Reconfiguring daemon mgr.smithi112.llioug on smithi112 2024-04-26T15:23:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:22 smithi164 ceph-mon[24045]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:22.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:22 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:22.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:22 smithi112 ceph-mon[21780]: Reconfiguring mgr.smithi112.llioug (unknown last config time)... 2024-04-26T15:23:22.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:22 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.llioug", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:23:22.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:22 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:23:22.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:22 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:22.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:22 smithi112 ceph-mon[21780]: Reconfiguring daemon mgr.smithi112.llioug on smithi112 2024-04-26T15:23:22.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:22 smithi112 ceph-mon[21780]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:23.513 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:23 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T15:23:23.513 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:23 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:23.513 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:23 smithi112 ceph-mon[21780]: Deploying daemon osd.0 on smithi112 2024-04-26T15:23:23.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:23 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T15:23:23.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:23 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:23.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:23 smithi164 ceph-mon[24045]: Deploying daemon osd.0 on smithi112 2024-04-26T15:23:25.178 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:24 smithi112 ceph-mon[21780]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:25.178 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:24 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:25.179 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:24 smithi112 ceph-mon[21780]: Reconfiguring alertmanager.smithi112 (dependencies changed)... 2024-04-26T15:23:25.179 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:24 smithi112 ceph-mon[21780]: Reconfiguring daemon alertmanager.smithi112 on smithi112 2024-04-26T15:23:25.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:24 smithi164 ceph-mon[24045]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:25.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:24 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:25.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:24 smithi164 ceph-mon[24045]: Reconfiguring alertmanager.smithi112 (dependencies changed)... 2024-04-26T15:23:25.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:24 smithi164 ceph-mon[24045]: Reconfiguring daemon alertmanager.smithi112 on smithi112 2024-04-26T15:23:26.865 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:26 smithi112 ceph-mon[21780]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:27.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:26 smithi164 ceph-mon[24045]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:28 smithi112 ceph-mon[21780]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:23:28.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:23:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:28 smithi164 ceph-mon[24045]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:23:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:23:30.338 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:30 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:30.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:30 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:31.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:31 smithi164 ceph-mon[24045]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:31.355 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:31 smithi112 ceph-mon[21780]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:33.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:33 smithi164 ceph-mon[24045]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:33.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:33 smithi164 ceph-mon[24045]: from='osd.0 [v2:172.21.15.112:6802/2249725715,v1:172.21.15.112:6803/2249725715]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T15:23:33.422 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:33 smithi112 ceph-mon[21780]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:33.422 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:33 smithi112 ceph-mon[21780]: from='osd.0 [v2:172.21.15.112:6802/2249725715,v1:172.21.15.112:6803/2249725715]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T15:23:34.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:34 smithi164 ceph-mon[24045]: from='osd.0 [v2:172.21.15.112:6802/2249725715,v1:172.21.15.112:6803/2249725715]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T15:23:34.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:34 smithi164 ceph-mon[24045]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T15:23:34.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:34 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:23:34.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:34 smithi164 ceph-mon[24045]: from='osd.0 [v2:172.21.15.112:6802/2249725715,v1:172.21.15.112:6803/2249725715]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:23:34.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:34 smithi112 ceph-mon[21780]: from='osd.0 [v2:172.21.15.112:6802/2249725715,v1:172.21.15.112:6803/2249725715]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T15:23:34.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:34 smithi112 ceph-mon[21780]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T15:23:34.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:34 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:23:34.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:34 smithi112 ceph-mon[21780]: from='osd.0 [v2:172.21.15.112:6802/2249725715,v1:172.21.15.112:6803/2249725715]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:23:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:35 smithi164 ceph-mon[24045]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:35 smithi164 ceph-mon[24045]: from='osd.0 [v2:172.21.15.112:6802/2249725715,v1:172.21.15.112:6803/2249725715]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-26T15:23:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:35 smithi164 ceph-mon[24045]: osdmap e8: 1 total, 0 up, 1 in 2024-04-26T15:23:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:23:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:23:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:23:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:35.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:35 smithi112 ceph-mon[21780]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:35.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:35 smithi112 ceph-mon[21780]: from='osd.0 [v2:172.21.15.112:6802/2249725715,v1:172.21.15.112:6803/2249725715]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-26T15:23:35.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:35 smithi112 ceph-mon[21780]: osdmap e8: 1 total, 0 up, 1 in 2024-04-26T15:23:35.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:23:35.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:23:35.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:35.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:23:35.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:36.173 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:36 smithi112 ceph-mon[21780]: purged_snaps scrub starts 2024-04-26T15:23:36.173 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:36 smithi112 ceph-mon[21780]: purged_snaps scrub ok 2024-04-26T15:23:36.173 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:36 smithi112 ceph-mon[21780]: Reconfiguring crash.smithi112 (monmap changed)... 2024-04-26T15:23:36.173 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:36 smithi112 ceph-mon[21780]: Reconfiguring daemon crash.smithi112 on smithi112 2024-04-26T15:23:36.173 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:23:36.173 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:36 smithi112 ceph-mon[21780]: osd.0 [v2:172.21.15.112:6802/2249725715,v1:172.21.15.112:6803/2249725715] boot 2024-04-26T15:23:36.174 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:36 smithi112 ceph-mon[21780]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T15:23:36.174 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:23:36.204 INFO:teuthology.orchestra.run.smithi112.stdout:Created osd(s) 0 on host 'smithi112' 2024-04-26T15:23:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:36 smithi164 ceph-mon[24045]: purged_snaps scrub starts 2024-04-26T15:23:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:36 smithi164 ceph-mon[24045]: purged_snaps scrub ok 2024-04-26T15:23:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:36 smithi164 ceph-mon[24045]: Reconfiguring crash.smithi112 (monmap changed)... 2024-04-26T15:23:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:36 smithi164 ceph-mon[24045]: Reconfiguring daemon crash.smithi112 on smithi112 2024-04-26T15:23:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:36 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:23:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:36 smithi164 ceph-mon[24045]: osd.0 [v2:172.21.15.112:6802/2249725715,v1:172.21.15.112:6803/2249725715] boot 2024-04-26T15:23:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:36 smithi164 ceph-mon[24045]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T15:23:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:36 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:23:36.774 DEBUG:teuthology.orchestra.run.smithi112:osd.0> sudo journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.0.service 2024-04-26T15:23:36.776 INFO:tasks.cephadm:Deploying osd.1 on smithi112 with /dev/vg_nvme/lv_3... 2024-04-26T15:23:36.776 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T15:23:37.255 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:37 smithi112 ceph-mon[21780]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:37.255 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:37 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:37.255 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:37 smithi112 ceph-mon[21780]: osdmap e10: 1 total, 1 up, 1 in 2024-04-26T15:23:37.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:37 smithi164 ceph-mon[24045]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:23:37.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:37 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:37.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:37 smithi164 ceph-mon[24045]: osdmap e10: 1 total, 1 up, 1 in 2024-04-26T15:23:39.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:39 smithi112 ceph-mon[21780]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:39.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:39 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:39.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:39 smithi164 ceph-mon[24045]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:39.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:39 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:40.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:40 smithi112 ceph-mon[21780]: Reconfiguring grafana.smithi112 (dependencies changed)... 2024-04-26T15:23:40.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:40 smithi112 ceph-mon[21780]: Reconfiguring daemon grafana.smithi112 on smithi112 2024-04-26T15:23:40.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:40 smithi164 ceph-mon[24045]: Reconfiguring grafana.smithi112 (dependencies changed)... 2024-04-26T15:23:40.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:40 smithi164 ceph-mon[24045]: Reconfiguring daemon grafana.smithi112 on smithi112 2024-04-26T15:23:41.382 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T15:23:41.382 INFO:teuthology.orchestra.run.smithi112.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T15:23:41.383 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 10+0 records in 2024-04-26T15:23:41.383 INFO:teuthology.orchestra.run.smithi112.stderr:10+0 records out 2024-04-26T15:23:41.383 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0144137 s, 727 MB/s 2024-04-26T15:23:41.383 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping successful for: 2024-04-26T15:23:41.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:41 smithi164 ceph-mon[24045]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:41.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:41 smithi112 ceph-mon[21780]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:41.965 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi112:vg_nvme/lv_3 2024-04-26T15:23:43.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:43 smithi112 ceph-mon[21780]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:43.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:43 smithi164 ceph-mon[24045]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:45.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:45 smithi112 ceph-mon[21780]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:45.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:45 smithi164 ceph-mon[24045]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:46.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:46 smithi112 ceph-mon[21780]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:46.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:46 smithi112 ceph-mon[21780]: from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:23:46.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:46 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:23:46.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:46 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:23:46.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:46 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:46.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:46 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:46.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:46 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:23:46.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:46 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:46.925 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:46 smithi164 ceph-mon[24045]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:46.925 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:46 smithi164 ceph-mon[24045]: from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:23:46.925 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:46 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:23:46.925 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:46 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:23:46.925 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:46 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:46.925 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:46 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:46.925 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:46 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:23:46.925 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:46 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:47.941 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:47 smithi164 ceph-mon[24045]: Reconfiguring crash.smithi164 (monmap changed)... 2024-04-26T15:23:47.941 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:47 smithi164 ceph-mon[24045]: Reconfiguring daemon crash.smithi164 on smithi164 2024-04-26T15:23:47.957 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:47 smithi112 ceph-mon[21780]: Reconfiguring crash.smithi164 (monmap changed)... 2024-04-26T15:23:47.957 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:47 smithi112 ceph-mon[21780]: Reconfiguring daemon crash.smithi164 on smithi164 2024-04-26T15:23:49.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:48 smithi164 ceph-mon[24045]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:49.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:48 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:49.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:48 smithi164 ceph-mon[24045]: Reconfiguring mgr.smithi164.tuhwus (monmap changed)... 2024-04-26T15:23:49.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:48 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.tuhwus", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:23:49.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:48 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:23:49.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:48 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:49.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:48 smithi164 ceph-mon[24045]: Reconfiguring daemon mgr.smithi164.tuhwus on smithi164 2024-04-26T15:23:49.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:48 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/4196218243' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "baa30761-660d-452a-bdfe-00569457e9a1"}]: dispatch 2024-04-26T15:23:49.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:48 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/4196218243' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "baa30761-660d-452a-bdfe-00569457e9a1"}]': finished 2024-04-26T15:23:49.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:48 smithi164 ceph-mon[24045]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T15:23:49.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:48 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:23:49.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:48 smithi112 ceph-mon[21780]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:49.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:48 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:49.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:48 smithi112 ceph-mon[21780]: Reconfiguring mgr.smithi164.tuhwus (monmap changed)... 2024-04-26T15:23:49.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:48 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.tuhwus", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:23:49.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:48 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:23:49.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:48 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:49.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:48 smithi112 ceph-mon[21780]: Reconfiguring daemon mgr.smithi164.tuhwus on smithi164 2024-04-26T15:23:49.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:48 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/4196218243' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "baa30761-660d-452a-bdfe-00569457e9a1"}]: dispatch 2024-04-26T15:23:49.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:48 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/4196218243' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "baa30761-660d-452a-bdfe-00569457e9a1"}]': finished 2024-04-26T15:23:49.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:48 smithi112 ceph-mon[21780]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T15:23:49.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:48 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:23:50.437 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:50.437 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:50 smithi164 ceph-mon[24045]: Reconfiguring mon.smithi164 (monmap changed)... 2024-04-26T15:23:50.437 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:23:50.438 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:23:50.438 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:50.438 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:50 smithi164 ceph-mon[24045]: Reconfiguring daemon mon.smithi164 on smithi164 2024-04-26T15:23:50.438 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:50 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1065008463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:23:50.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:50.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:50 smithi112 ceph-mon[21780]: Reconfiguring mon.smithi164 (monmap changed)... 2024-04-26T15:23:50.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:23:50.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:23:50.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:50.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:50 smithi112 ceph-mon[21780]: Reconfiguring daemon mon.smithi164 on smithi164 2024-04-26T15:23:50.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:50 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1065008463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:23:51.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:51 smithi112 ceph-mon[21780]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:51.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:51 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:51.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:51 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:23:51.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:51 smithi164 ceph-mon[24045]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:51.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:51 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:23:51.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:51 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:23:52.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:52 smithi112 ceph-mon[21780]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:23:52.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:52 smithi164 ceph-mon[24045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:23:53.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:53 smithi112 ceph-mon[21780]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:53.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:53 smithi164 ceph-mon[24045]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:55.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:55 smithi112 ceph-mon[21780]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:55.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:55 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T15:23:55.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:55 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:55.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:55 smithi164 ceph-mon[24045]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:55.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:55 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T15:23:55.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:55 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:23:56.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:56 smithi112 ceph-mon[21780]: Deploying daemon osd.1 on smithi112 2024-04-26T15:23:56.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:56 smithi164 ceph-mon[24045]: Deploying daemon osd.1 on smithi112 2024-04-26T15:23:57.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:57 smithi112 ceph-mon[21780]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:57.574 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:57 smithi164 ceph-mon[24045]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:23:59 smithi112 ceph-mon[21780]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:23:59.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:23:59 smithi164 ceph-mon[24045]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:24:01.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:01 smithi164 ceph-mon[24045]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:24:01.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:01 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:01.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:01 smithi112 ceph-mon[21780]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:24:01.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:01 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:02.929 INFO:teuthology.orchestra.run.smithi112.stdout:Created osd(s) 1 on host 'smithi112' 2024-04-26T15:24:03.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:24:03.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:03.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.112:9093"}]: dispatch 2024-04-26T15:24:03.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:03.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:24:03.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.112:3000"}]: dispatch 2024-04-26T15:24:03.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:03.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:24:03.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.112:9095"}]: dispatch 2024-04-26T15:24:03.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:03.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:24:03.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:03.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:24:03.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:24:03.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:03.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.112:9093"}]: dispatch 2024-04-26T15:24:03.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:03.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:24:03.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.112:3000"}]: dispatch 2024-04-26T15:24:03.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:03.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:24:03.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.112:9095"}]: dispatch 2024-04-26T15:24:03.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:03.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:24:03.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:03.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:24:03.541 DEBUG:teuthology.orchestra.run.smithi112:osd.1> sudo journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.1.service 2024-04-26T15:24:03.544 INFO:tasks.cephadm:Deploying osd.2 on smithi112 with /dev/vg_nvme/lv_2... 2024-04-26T15:24:03.544 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T15:24:04.164 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:04 smithi112 ceph-mon[21780]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.112:9093"}]: dispatch 2024-04-26T15:24:04.164 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:04 smithi112 ceph-mon[21780]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:24:04.164 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:04 smithi112 ceph-mon[21780]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.112:3000"}]: dispatch 2024-04-26T15:24:04.164 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:04 smithi112 ceph-mon[21780]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:24:04.164 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:04 smithi112 ceph-mon[21780]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.112:9095"}]: dispatch 2024-04-26T15:24:04.164 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:04 smithi112 ceph-mon[21780]: from='osd.1 [v2:172.21.15.112:6810/20141175,v1:172.21.15.112:6811/20141175]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T15:24:04.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:04 smithi164 ceph-mon[24045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.112:9093"}]: dispatch 2024-04-26T15:24:04.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:04 smithi164 ceph-mon[24045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:24:04.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:04 smithi164 ceph-mon[24045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.112:3000"}]: dispatch 2024-04-26T15:24:04.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:04 smithi164 ceph-mon[24045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:24:04.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:04 smithi164 ceph-mon[24045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.112:9095"}]: dispatch 2024-04-26T15:24:04.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:04 smithi164 ceph-mon[24045]: from='osd.1 [v2:172.21.15.112:6810/20141175,v1:172.21.15.112:6811/20141175]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T15:24:05.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:05 smithi164 ceph-mon[24045]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:24:05.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:05 smithi164 ceph-mon[24045]: from='osd.1 [v2:172.21.15.112:6810/20141175,v1:172.21.15.112:6811/20141175]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T15:24:05.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:05 smithi164 ceph-mon[24045]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T15:24:05.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:05 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:24:05.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:05 smithi164 ceph-mon[24045]: from='osd.1 [v2:172.21.15.112:6810/20141175,v1:172.21.15.112:6811/20141175]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:24:05.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:05 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:05.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:05 smithi112 ceph-mon[21780]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:24:05.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:05 smithi112 ceph-mon[21780]: from='osd.1 [v2:172.21.15.112:6810/20141175,v1:172.21.15.112:6811/20141175]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T15:24:05.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:05 smithi112 ceph-mon[21780]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T15:24:05.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:05 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:24:05.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:05 smithi112 ceph-mon[21780]: from='osd.1 [v2:172.21.15.112:6810/20141175,v1:172.21.15.112:6811/20141175]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:24:05.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:05 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:05.427 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:24:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[35107]: 2024-04-26T15:24:05.096+0000 7f2451b44700 -1 osd.1 0 waiting for initial osdmap 2024-04-26T15:24:05.427 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:24:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[35107]: 2024-04-26T15:24:05.099+0000 7f244c4db700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:24:06.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:06 smithi112 ceph-mon[21780]: from='osd.1 [v2:172.21.15.112:6810/20141175,v1:172.21.15.112:6811/20141175]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-26T15:24:06.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:06 smithi112 ceph-mon[21780]: osdmap e13: 2 total, 1 up, 2 in 2024-04-26T15:24:06.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:06 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:24:06.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:06 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:24:06.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:06 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:24:06.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:06 smithi164 ceph-mon[24045]: from='osd.1 [v2:172.21.15.112:6810/20141175,v1:172.21.15.112:6811/20141175]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-26T15:24:06.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:06 smithi164 ceph-mon[24045]: osdmap e13: 2 total, 1 up, 2 in 2024-04-26T15:24:06.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:06 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:24:06.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:06 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:24:06.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:06 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:24:07.137 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T15:24:07.138 INFO:teuthology.orchestra.run.smithi112.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T15:24:07.138 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 10+0 records in 2024-04-26T15:24:07.138 INFO:teuthology.orchestra.run.smithi112.stderr:10+0 records out 2024-04-26T15:24:07.138 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124562 s, 842 MB/s 2024-04-26T15:24:07.138 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping successful for: 2024-04-26T15:24:07.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:07 smithi112 ceph-mon[21780]: purged_snaps scrub starts 2024-04-26T15:24:07.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:07 smithi112 ceph-mon[21780]: purged_snaps scrub ok 2024-04-26T15:24:07.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:07 smithi112 ceph-mon[21780]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:24:07.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:07 smithi112 ceph-mon[21780]: osd.1 [v2:172.21.15.112:6810/20141175,v1:172.21.15.112:6811/20141175] boot 2024-04-26T15:24:07.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:07 smithi112 ceph-mon[21780]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T15:24:07.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:07 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:24:07.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:07 smithi164 ceph-mon[24045]: purged_snaps scrub starts 2024-04-26T15:24:07.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:07 smithi164 ceph-mon[24045]: purged_snaps scrub ok 2024-04-26T15:24:07.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:07 smithi164 ceph-mon[24045]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:24:07.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:07 smithi164 ceph-mon[24045]: osd.1 [v2:172.21.15.112:6810/20141175,v1:172.21.15.112:6811/20141175] boot 2024-04-26T15:24:07.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:07 smithi164 ceph-mon[24045]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T15:24:07.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:07 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:24:08.287 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:08 smithi112 ceph-mon[21780]: osdmap e15: 2 total, 2 up, 2 in 2024-04-26T15:24:08.311 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi112:vg_nvme/lv_2 2024-04-26T15:24:08.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:08 smithi164 ceph-mon[24045]: osdmap e15: 2 total, 2 up, 2 in 2024-04-26T15:24:09.454 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:09 smithi112 ceph-mon[21780]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:09.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:09 smithi164 ceph-mon[24045]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:11.202 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:11 smithi112 ceph-mon[21780]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:11.202 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:11 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:24:11.202 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:11 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:24:11.203 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:11 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:11.203 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:11 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:11.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:11 smithi164 ceph-mon[24045]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:11.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:11 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:24:11.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:11 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:24:11.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:11 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:11.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:11 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:12.414 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:12 smithi112 ceph-mon[21780]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:24:12.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:12 smithi164 ceph-mon[24045]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:24:13.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:13 smithi112 ceph-mon[21780]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:13.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:13 smithi164 ceph-mon[24045]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:14.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:14 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1156854481' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17e98a3f-1edf-4186-8a34-2dce3e86d234"}]: dispatch 2024-04-26T15:24:14.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:14 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1156854481' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "17e98a3f-1edf-4186-8a34-2dce3e86d234"}]': finished 2024-04-26T15:24:14.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:14 smithi112 ceph-mon[21780]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T15:24:14.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:24:14.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:14 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1156854481' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17e98a3f-1edf-4186-8a34-2dce3e86d234"}]: dispatch 2024-04-26T15:24:14.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:14 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1156854481' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "17e98a3f-1edf-4186-8a34-2dce3e86d234"}]': finished 2024-04-26T15:24:14.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:14 smithi164 ceph-mon[24045]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T15:24:14.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:24:15.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:15 smithi112 ceph-mon[21780]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:15.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:15 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/3799652722' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:24:15.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:15 smithi164 ceph-mon[24045]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:15.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:15 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/3799652722' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:24:17.400 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:17 smithi164 ceph-mon[24045]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:17.440 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:17 smithi112 ceph-mon[21780]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:19.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:19 smithi112 ceph-mon[21780]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:19.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:19 smithi164 ceph-mon[24045]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:20.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:20 smithi164 ceph-mon[24045]: Detected new or changed devices on smithi112 2024-04-26T15:24:20.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:20.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:24:20.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:24:20.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:20 smithi164 ceph-mon[24045]: Adjusting osd_memory_target on smithi112 to 6432M 2024-04-26T15:24:20.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:20.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:20.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:24:20.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:20.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:24:20.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:20 smithi164 ceph-mon[24045]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:20.925 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:20 smithi112 ceph-mon[21780]: Detected new or changed devices on smithi112 2024-04-26T15:24:20.925 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:20.925 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:24:20.925 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:24:20.925 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:20 smithi112 ceph-mon[21780]: Adjusting osd_memory_target on smithi112 to 6432M 2024-04-26T15:24:20.925 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:20.925 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:20.925 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:24:20.925 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:20.926 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:24:20.926 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:20 smithi112 ceph-mon[21780]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:22 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:22 smithi112 ceph-mon[21780]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:22 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T15:24:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:22 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:22 smithi112 ceph-mon[21780]: Deploying daemon osd.2 on smithi112 2024-04-26T15:24:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:22 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:22 smithi164 ceph-mon[24045]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:22 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T15:24:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:22 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:22 smithi164 ceph-mon[24045]: Deploying daemon osd.2 on smithi112 2024-04-26T15:24:25.012 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:24 smithi112 ceph-mon[21780]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:24 smithi164 ceph-mon[24045]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:26.911 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:26 smithi112 ceph-mon[21780]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:27.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:26 smithi164 ceph-mon[24045]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:28.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:28.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:28 smithi112 ceph-mon[21780]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:28.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:24:28.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:24:29.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:29.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:28 smithi164 ceph-mon[24045]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:29.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:24:29.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:24:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:31 smithi112 ceph-mon[21780]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:31.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:31 smithi164 ceph-mon[24045]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:31.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:33.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:33 smithi164 ceph-mon[24045]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:33.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:33 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:33.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:33 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:33.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:33 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:24:33.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:33 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:33.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:33 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:24:33.357 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:33 smithi112 ceph-mon[21780]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:33.357 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:33 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:33.357 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:33 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:33.357 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:33 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:24:33.357 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:33 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:33.357 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:33 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:24:34.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:34 smithi164 ceph-mon[24045]: from='osd.2 [v2:172.21.15.112:6818/1768051982,v1:172.21.15.112:6819/1768051982]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T15:24:34.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:34 smithi164 ceph-mon[24045]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T15:24:34.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:34 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:34.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:34 smithi112 ceph-mon[21780]: from='osd.2 [v2:172.21.15.112:6818/1768051982,v1:172.21.15.112:6819/1768051982]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T15:24:34.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:34 smithi112 ceph-mon[21780]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T15:24:34.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:34 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:34.567 INFO:teuthology.orchestra.run.smithi112.stdout:Created osd(s) 2 on host 'smithi112' 2024-04-26T15:24:35.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:35 smithi112 ceph-mon[21780]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:35.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:35 smithi112 ceph-mon[21780]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T15:24:35.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:35 smithi112 ceph-mon[21780]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T15:24:35.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:24:35.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:35 smithi112 ceph-mon[21780]: from='osd.2 [v2:172.21.15.112:6818/1768051982,v1:172.21.15.112:6819/1768051982]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:24:35.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:35 smithi112 ceph-mon[21780]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:24:35.188 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:35.211 DEBUG:teuthology.orchestra.run.smithi112:osd.2> sudo journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.2.service 2024-04-26T15:24:35.213 INFO:tasks.cephadm:Deploying osd.3 on smithi112 with /dev/vg_nvme/lv_1... 2024-04-26T15:24:35.214 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T15:24:35.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:35 smithi164 ceph-mon[24045]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:35.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:35 smithi164 ceph-mon[24045]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T15:24:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:35 smithi164 ceph-mon[24045]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T15:24:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:24:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:35 smithi164 ceph-mon[24045]: from='osd.2 [v2:172.21.15.112:6818/1768051982,v1:172.21.15.112:6819/1768051982]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:24:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:35 smithi164 ceph-mon[24045]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:24:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:36.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:36 smithi112 ceph-mon[21780]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-26T15:24:36.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:36 smithi112 ceph-mon[21780]: osdmap e18: 3 total, 2 up, 3 in 2024-04-26T15:24:36.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:24:36.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:24:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:36 smithi164 ceph-mon[24045]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-26T15:24:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:36 smithi164 ceph-mon[24045]: osdmap e18: 3 total, 2 up, 3 in 2024-04-26T15:24:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:36 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:24:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:36 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:24:37.303 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:37 smithi112 ceph-mon[21780]: purged_snaps scrub starts 2024-04-26T15:24:37.303 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:37 smithi112 ceph-mon[21780]: purged_snaps scrub ok 2024-04-26T15:24:37.303 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:37 smithi112 ceph-mon[21780]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:37.303 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:37 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:24:37.303 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:37 smithi112 ceph-mon[21780]: osd.2 [v2:172.21.15.112:6818/1768051982,v1:172.21.15.112:6819/1768051982] boot 2024-04-26T15:24:37.304 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:37 smithi112 ceph-mon[21780]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T15:24:37.304 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:37 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:24:37.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:37 smithi164 ceph-mon[24045]: purged_snaps scrub starts 2024-04-26T15:24:37.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:37 smithi164 ceph-mon[24045]: purged_snaps scrub ok 2024-04-26T15:24:37.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:37 smithi164 ceph-mon[24045]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:24:37.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:37 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:24:37.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:37 smithi164 ceph-mon[24045]: osd.2 [v2:172.21.15.112:6818/1768051982,v1:172.21.15.112:6819/1768051982] boot 2024-04-26T15:24:37.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:37 smithi164 ceph-mon[24045]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T15:24:37.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:37 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:24:37.776 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T15:24:37.776 INFO:teuthology.orchestra.run.smithi112.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T15:24:37.776 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 10+0 records in 2024-04-26T15:24:37.776 INFO:teuthology.orchestra.run.smithi112.stderr:10+0 records out 2024-04-26T15:24:37.776 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0121485 s, 863 MB/s 2024-04-26T15:24:37.776 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping successful for: 2024-04-26T15:24:38.328 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:38 smithi112 ceph-mon[21780]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T15:24:38.328 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:38 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T15:24:38.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:38 smithi164 ceph-mon[24045]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T15:24:38.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:38 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T15:24:38.925 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi112:vg_nvme/lv_1 2024-04-26T15:24:39.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:39 smithi112 ceph-mon[21780]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:24:39.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:39 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T15:24:39.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:39 smithi112 ceph-mon[21780]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T15:24:39.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:39 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T15:24:39.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:39 smithi164 ceph-mon[24045]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:24:39.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:39 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T15:24:39.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:39 smithi164 ceph-mon[24045]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T15:24:39.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:39 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T15:24:40.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:40 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T15:24:40.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:40 smithi164 ceph-mon[24045]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T15:24:40.404 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:40 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T15:24:40.404 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:40 smithi112 ceph-mon[21780]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T15:24:40.405 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:24:40 smithi112 sudo[40353]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T15:24:40.405 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:24:40 smithi112 sudo[40353]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:24:40.405 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:24:40 smithi112 sudo[40353]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:24:40.670 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:24:40 smithi112 sudo[40356]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T15:24:40.670 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:24:40 smithi112 sudo[40356]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:24:40.670 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:24:40 smithi112 sudo[40356]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:24:40.938 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:24:40 smithi112 sudo[40381]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T15:24:40.939 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:24:40 smithi112 sudo[40381]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:24:40.939 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:24:40 smithi112 sudo[40381]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:24:41.189 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:41 smithi112 ceph-mon[21780]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:24:41.190 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:41 smithi112 ceph-mon[21780]: osdmap e23: 3 total, 3 up, 3 in 2024-04-26T15:24:41.190 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:41 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:41.190 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:41 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:24:41.190 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:41 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:24:41.190 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:41 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:41.190 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:24:40 smithi112 sudo[40407]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T15:24:41.190 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:24:40 smithi112 sudo[40407]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:24:41.190 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:24:40 smithi112 sudo[40407]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:24:41.453 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:24:41 smithi112 sudo[40448]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T15:24:41.453 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:24:41 smithi112 sudo[40448]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:24:41.453 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:24:41 smithi112 sudo[40448]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:24:41.453 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:24:41 smithi112 sudo[40520]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T15:24:41.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:41 smithi164 ceph-mon[24045]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:24:41.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:41 smithi164 ceph-mon[24045]: osdmap e23: 3 total, 3 up, 3 in 2024-04-26T15:24:41.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:41 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:41.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:41 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:24:41.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:41 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:24:41.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:41 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:41.711 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:24:41 smithi112 sudo[40520]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:24:41.711 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:24:41 smithi112 sudo[40520]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:24:41.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:41 smithi112 sudo[40552]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T15:24:41.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:41 smithi112 sudo[40552]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:24:42.090 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:41 smithi112 sudo[40552]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:24:42.090 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:41 smithi112 sudo[40559]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T15:24:42.090 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:41 smithi112 sudo[40559]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:24:42.090 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:41 smithi112 sudo[40559]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:24:42.456 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 sudo[26368]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T15:24:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 sudo[26368]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:24:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 sudo[26368]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:24:42.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:42 smithi112 ceph-mon[21780]: from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:24:42.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:42 smithi112 ceph-mon[21780]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:24:42.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:42 smithi112 ceph-mon[21780]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:24:42.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:24:42.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:24:42.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:24:42.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:24:42.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 sudo[26371]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T15:24:42.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 sudo[26371]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:24:42.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 sudo[26371]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:24:42.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 ceph-mon[24045]: from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:24:42.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 ceph-mon[24045]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:24:42.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 ceph-mon[24045]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:24:42.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:24:42.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:24:42.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:24:42.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:42 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:24:43.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:43 smithi164 ceph-mon[24045]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:24:43.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:43 smithi164 ceph-mon[24045]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:24:43.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:43 smithi164 ceph-mon[24045]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:24:43.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:43 smithi164 ceph-mon[24045]: mgrmap e19: smithi112.llioug(active, since 2m), standbys: smithi164.tuhwus 2024-04-26T15:24:43.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:43 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:43.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:43 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:43.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:43 smithi112 ceph-mon[21780]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:24:43.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:43 smithi112 ceph-mon[21780]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:24:43.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:43 smithi112 ceph-mon[21780]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:24:43.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:43 smithi112 ceph-mon[21780]: mgrmap e19: smithi112.llioug(active, since 2m), standbys: smithi164.tuhwus 2024-04-26T15:24:43.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:43 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:43.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:43 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:44.801 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:44 smithi112 ceph-mon[21780]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:24:44.801 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:44 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2776834718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c85a093-d36e-4c61-bd75-95c2f09be643"}]: dispatch 2024-04-26T15:24:44.801 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:44 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2776834718' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6c85a093-d36e-4c61-bd75-95c2f09be643"}]': finished 2024-04-26T15:24:44.801 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:44 smithi112 ceph-mon[21780]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T15:24:44.801 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:24:44.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:44 smithi164 ceph-mon[24045]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:24:44.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:44 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/2776834718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c85a093-d36e-4c61-bd75-95c2f09be643"}]: dispatch 2024-04-26T15:24:44.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:44 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/2776834718' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6c85a093-d36e-4c61-bd75-95c2f09be643"}]': finished 2024-04-26T15:24:44.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:44 smithi164 ceph-mon[24045]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T15:24:44.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:24:45.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:45 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1337487806' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:24:45.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:45 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1337487806' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:24:46.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:46 smithi112 ceph-mon[21780]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:24:47.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:46 smithi164 ceph-mon[24045]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:24:48.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:48 smithi112 ceph-mon[21780]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-26T15:24:49.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:48 smithi164 ceph-mon[24045]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-26T15:24:50.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:50 smithi112 ceph-mon[21780]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-26T15:24:50.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:50.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:24:50.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:24:50.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:24:50.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:50.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:50.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:24:50.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:50.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:50 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:24:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:50 smithi164 ceph-mon[24045]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-26T15:24:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:24:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:24:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:24:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:24:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:50 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:24:51.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:51 smithi112 ceph-mon[21780]: Detected new or changed devices on smithi112 2024-04-26T15:24:51.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:51 smithi112 ceph-mon[21780]: Adjusting osd_memory_target on smithi112 to 4288M 2024-04-26T15:24:52.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:51 smithi164 ceph-mon[24045]: Detected new or changed devices on smithi112 2024-04-26T15:24:52.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:51 smithi164 ceph-mon[24045]: Adjusting osd_memory_target on smithi112 to 4288M 2024-04-26T15:24:52.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:52 smithi112 ceph-mon[21780]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T15:24:52.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:52 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T15:24:52.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:52 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:52.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:52 smithi112 ceph-mon[21780]: Deploying daemon osd.3 on smithi112 2024-04-26T15:24:53.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:52 smithi164 ceph-mon[24045]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T15:24:53.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:52 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T15:24:53.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:52 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:24:53.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:52 smithi164 ceph-mon[24045]: Deploying daemon osd.3 on smithi112 2024-04-26T15:24:54.858 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:54 smithi112 ceph-mon[21780]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T15:24:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:54 smithi164 ceph-mon[24045]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T15:24:56.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:56 smithi112 ceph-mon[21780]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 68 KiB/s wr, 4 op/s 2024-04-26T15:24:57.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:56 smithi164 ceph-mon[24045]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 68 KiB/s wr, 4 op/s 2024-04-26T15:24:58.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:58 smithi112 ceph-mon[21780]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T15:24:58.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:58 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:58.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:24:58 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:59.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:58 smithi164 ceph-mon[24045]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T15:24:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:58 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:24:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:24:58 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:00.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:00.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:25:00.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:00.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:25:00.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:00 smithi112 ceph-mon[21780]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:25:00.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:00.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:25:00.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:00.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:25:00.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:00 smithi164 ceph-mon[24045]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:25:02.955 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:02 smithi112 ceph-mon[21780]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:25:02.955 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:02 smithi112 ceph-mon[21780]: from='osd.3 [v2:172.21.15.112:6826/1408970703,v1:172.21.15.112:6827/1408970703]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T15:25:02.988 INFO:teuthology.orchestra.run.smithi112.stdout:Created osd(s) 3 on host 'smithi112' 2024-04-26T15:25:03.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:02 smithi164 ceph-mon[24045]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:25:03.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:02 smithi164 ceph-mon[24045]: from='osd.3 [v2:172.21.15.112:6826/1408970703,v1:172.21.15.112:6827/1408970703]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T15:25:03.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:03 smithi112 ceph-mon[21780]: from='osd.3 [v2:172.21.15.112:6826/1408970703,v1:172.21.15.112:6827/1408970703]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T15:25:03.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:03 smithi112 ceph-mon[21780]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T15:25:03.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:25:03.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:03 smithi112 ceph-mon[21780]: from='osd.3 [v2:172.21.15.112:6826/1408970703,v1:172.21.15.112:6827/1408970703]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:25:03.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:03.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:03 smithi112 ceph-mon[21780]: from='osd.3 [v2:172.21.15.112:6826/1408970703,v1:172.21.15.112:6827/1408970703]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-26T15:25:03.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:03 smithi112 ceph-mon[21780]: osdmap e26: 4 total, 3 up, 4 in 2024-04-26T15:25:03.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:03 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:25:03.863 DEBUG:teuthology.orchestra.run.smithi112:osd.3> sudo journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.3.service 2024-04-26T15:25:03.865 INFO:tasks.cephadm:Deploying osd.4 on smithi164 with /dev/vg_nvme/lv_4... 2024-04-26T15:25:03.865 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T15:25:04.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:03 smithi164 ceph-mon[24045]: from='osd.3 [v2:172.21.15.112:6826/1408970703,v1:172.21.15.112:6827/1408970703]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T15:25:04.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:03 smithi164 ceph-mon[24045]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T15:25:04.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:25:04.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:03 smithi164 ceph-mon[24045]: from='osd.3 [v2:172.21.15.112:6826/1408970703,v1:172.21.15.112:6827/1408970703]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:25:04.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:04.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:03 smithi164 ceph-mon[24045]: from='osd.3 [v2:172.21.15.112:6826/1408970703,v1:172.21.15.112:6827/1408970703]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-26T15:25:04.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:03 smithi164 ceph-mon[24045]: osdmap e26: 4 total, 3 up, 4 in 2024-04-26T15:25:04.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:03 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:25:04.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:04 smithi164 ceph-mon[24045]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:25:04.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:04 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:25:04.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:04 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:25:05.092 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:04 smithi112 ceph-mon[21780]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:25:05.093 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:04 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:25:05.093 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:04 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:25:05.957 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T15:25:05.957 INFO:teuthology.orchestra.run.smithi164.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T15:25:05.957 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10+0 records in 2024-04-26T15:25:05.958 INFO:teuthology.orchestra.run.smithi164.stderr:10+0 records out 2024-04-26T15:25:05.958 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0137644 s, 762 MB/s 2024-04-26T15:25:05.958 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping successful for: 2024-04-26T15:25:05.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:05 smithi112 ceph-mon[21780]: purged_snaps scrub starts 2024-04-26T15:25:05.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:05 smithi112 ceph-mon[21780]: purged_snaps scrub ok 2024-04-26T15:25:05.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:05 smithi112 ceph-mon[21780]: osd.3 [v2:172.21.15.112:6826/1408970703,v1:172.21.15.112:6827/1408970703] boot 2024-04-26T15:25:05.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:05 smithi112 ceph-mon[21780]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T15:25:05.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:05 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:25:05.978 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:05 smithi164 ceph-mon[24045]: purged_snaps scrub starts 2024-04-26T15:25:05.978 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:05 smithi164 ceph-mon[24045]: purged_snaps scrub ok 2024-04-26T15:25:05.979 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:05 smithi164 ceph-mon[24045]: osd.3 [v2:172.21.15.112:6826/1408970703,v1:172.21.15.112:6827/1408970703] boot 2024-04-26T15:25:05.979 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:05 smithi164 ceph-mon[24045]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T15:25:05.979 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:05 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:25:06.597 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi164:vg_nvme/lv_4 2024-04-26T15:25:06.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:06 smithi164 ceph-mon[24045]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:25:06.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:06 smithi164 ceph-mon[24045]: osdmap e28: 4 total, 4 up, 4 in 2024-04-26T15:25:06.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:06 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:06.912 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:06 smithi112 ceph-mon[21780]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:25:06.913 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:06 smithi112 ceph-mon[21780]: osdmap e28: 4 total, 4 up, 4 in 2024-04-26T15:25:06.913 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:06 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:08.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:08 smithi164 ceph-mon[24045]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:08.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:08 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:25:08.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:08 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:25:08.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:08 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:08.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:08 smithi112 ceph-mon[21780]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:08.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:08 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:25:08.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:08 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:25:08.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:08 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:09.985 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:09 smithi112 ceph-mon[21780]: from='client.24139 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:25:09.985 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:09 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:09.985 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:09 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:09.986 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:09 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:09.986 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:09 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:09.986 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:09 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:09.986 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:09 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:09.986 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:09 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:09.986 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:09 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:25:09.986 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:09 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:09.986 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:09 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:25:09.987 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:09 smithi164 ceph-mon[24045]: from='client.24139 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:25:09.987 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:09 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:09.987 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:09 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:09.987 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:09 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:09.987 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:09 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:09.987 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:09 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:09.987 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:09 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:09.987 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:09 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:09.987 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:09 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:25:09.987 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:09 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:09.987 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:09 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:25:11.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:10 smithi164 ceph-mon[24045]: Adjusting osd_memory_target on smithi112 to 3216M 2024-04-26T15:25:11.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:10 smithi164 ceph-mon[24045]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:11.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:10 smithi112 ceph-mon[21780]: Adjusting osd_memory_target on smithi112 to 3216M 2024-04-26T15:25:11.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:10 smithi112 ceph-mon[21780]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:12.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:11 smithi164 ceph-mon[24045]: from='client.? 172.21.15.164:0/2595968612' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31d857fe-771a-474b-8faf-bd38342cf00f"}]: dispatch 2024-04-26T15:25:12.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:11 smithi164 ceph-mon[24045]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31d857fe-771a-474b-8faf-bd38342cf00f"}]: dispatch 2024-04-26T15:25:12.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:11 smithi164 ceph-mon[24045]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "31d857fe-771a-474b-8faf-bd38342cf00f"}]': finished 2024-04-26T15:25:12.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:11 smithi164 ceph-mon[24045]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T15:25:12.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:11 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:25:12.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:11 smithi164 ceph-mon[24045]: from='client.? 172.21.15.164:0/1464340933' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:25:12.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:11 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/2595968612' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31d857fe-771a-474b-8faf-bd38342cf00f"}]: dispatch 2024-04-26T15:25:12.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:11 smithi112 ceph-mon[21780]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31d857fe-771a-474b-8faf-bd38342cf00f"}]: dispatch 2024-04-26T15:25:12.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:11 smithi112 ceph-mon[21780]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "31d857fe-771a-474b-8faf-bd38342cf00f"}]': finished 2024-04-26T15:25:12.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:11 smithi112 ceph-mon[21780]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T15:25:12.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:11 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:25:12.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:11 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/1464340933' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:25:13.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:12 smithi164 ceph-mon[24045]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:13.157 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:12 smithi112 ceph-mon[21780]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:14.806 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:14.806 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:14 smithi164 ceph-mon[24045]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:14.806 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:14.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:14.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:14 smithi112 ceph-mon[21780]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:14.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:16.817 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:16 smithi164 ceph-mon[24045]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:17.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:16 smithi112 ceph-mon[21780]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:17.822 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:17 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T15:25:17.822 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:17 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:17.822 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:17 smithi164 ceph-mon[24045]: Deploying daemon osd.4 on smithi164 2024-04-26T15:25:18.056 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:17 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T15:25:18.056 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:17 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:18.056 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:17 smithi112 ceph-mon[21780]: Deploying daemon osd.4 on smithi164 2024-04-26T15:25:19.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:18 smithi164 ceph-mon[24045]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:19.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:18 smithi112 ceph-mon[21780]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:20.995 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:20 smithi164 ceph-mon[24045]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:21.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:20 smithi112 ceph-mon[21780]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:23.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:22 smithi164 ceph-mon[24045]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:23.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:22 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:23.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:22 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:25:23.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:22 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:23.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:22 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:25:23.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:22 smithi112 ceph-mon[21780]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:23.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:22 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:23.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:22 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:25:23.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:22 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:23.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:22 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:25:25.033 INFO:teuthology.orchestra.run.smithi164.stdout:Created osd(s) 4 on host 'smithi164' 2024-04-26T15:25:25.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:24 smithi164 ceph-mon[24045]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:25.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:24 smithi112 ceph-mon[21780]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:25.672 DEBUG:teuthology.orchestra.run.smithi164:osd.4> sudo journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.4.service 2024-04-26T15:25:25.674 INFO:tasks.cephadm:Deploying osd.5 on smithi164 with /dev/vg_nvme/lv_3... 2024-04-26T15:25:25.675 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T15:25:26.195 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:26 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:26.196 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:26 smithi164 ceph-mon[24045]: from='osd.4 [v2:172.21.15.164:6800/3452220802,v1:172.21.15.164:6801/3452220802]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:25:26.196 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:26 smithi164 ceph-mon[24045]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:25:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:26 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:26.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:26 smithi112 ceph-mon[21780]: from='osd.4 [v2:172.21.15.164:6800/3452220802,v1:172.21.15.164:6801/3452220802]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:25:26.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:26 smithi112 ceph-mon[21780]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:25:27.098 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:25:27 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[28308]: 2024-04-26T15:25:27.046+0000 7f77424b1700 -1 osd.4 0 waiting for initial osdmap 2024-04-26T15:25:27.098 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:25:27 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[28308]: 2024-04-26T15:25:27.052+0000 7f773ce48700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:25:27.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:27 smithi164 ceph-mon[24045]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:27.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:27 smithi164 ceph-mon[24045]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T15:25:27.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:27 smithi164 ceph-mon[24045]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T15:25:27.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:27 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:25:27.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:27 smithi164 ceph-mon[24045]: from='osd.4 [v2:172.21.15.164:6800/3452220802,v1:172.21.15.164:6801/3452220802]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:25:27.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:27 smithi164 ceph-mon[24045]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:25:27.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:27 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:27.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:27 smithi112 ceph-mon[21780]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:27.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:27 smithi112 ceph-mon[21780]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T15:25:27.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:27 smithi112 ceph-mon[21780]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T15:25:27.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:27 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:25:27.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:27 smithi112 ceph-mon[21780]: from='osd.4 [v2:172.21.15.164:6800/3452220802,v1:172.21.15.164:6801/3452220802]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:25:27.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:27 smithi112 ceph-mon[21780]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:25:27.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:27 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:28.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:28 smithi164 ceph-mon[24045]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-04-26T15:25:28.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:28 smithi164 ceph-mon[24045]: osdmap e31: 5 total, 4 up, 5 in 2024-04-26T15:25:28.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:25:28.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:25:28.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:25:28.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:25:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:28 smithi112 ceph-mon[21780]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-04-26T15:25:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:28 smithi112 ceph-mon[21780]: osdmap e31: 5 total, 4 up, 5 in 2024-04-26T15:25:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:25:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:25:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:25:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:25:29.307 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:29 smithi164 ceph-mon[24045]: purged_snaps scrub starts 2024-04-26T15:25:29.307 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:29 smithi164 ceph-mon[24045]: purged_snaps scrub ok 2024-04-26T15:25:29.307 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:29 smithi164 ceph-mon[24045]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:29.307 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:29 smithi164 ceph-mon[24045]: osd.4 [v2:172.21.15.164:6800/3452220802,v1:172.21.15.164:6801/3452220802] boot 2024-04-26T15:25:29.307 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:29 smithi164 ceph-mon[24045]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T15:25:29.307 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:29 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:25:29.307 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:29 smithi164 ceph-mon[24045]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T15:25:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:29 smithi112 ceph-mon[21780]: purged_snaps scrub starts 2024-04-26T15:25:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:29 smithi112 ceph-mon[21780]: purged_snaps scrub ok 2024-04-26T15:25:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:29 smithi112 ceph-mon[21780]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:25:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:29 smithi112 ceph-mon[21780]: osd.4 [v2:172.21.15.164:6800/3452220802,v1:172.21.15.164:6801/3452220802] boot 2024-04-26T15:25:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:29 smithi112 ceph-mon[21780]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T15:25:29.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:29 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:25:29.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:29 smithi112 ceph-mon[21780]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T15:25:29.580 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T15:25:29.580 INFO:teuthology.orchestra.run.smithi164.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T15:25:29.580 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10+0 records in 2024-04-26T15:25:29.580 INFO:teuthology.orchestra.run.smithi164.stderr:10+0 records out 2024-04-26T15:25:29.581 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0175451 s, 598 MB/s 2024-04-26T15:25:29.581 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping successful for: 2024-04-26T15:25:30.460 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi164:vg_nvme/lv_3 2024-04-26T15:25:31.327 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:31 smithi164 ceph-mon[24045]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:31.328 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:31 smithi164 ceph-mon[24045]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T15:25:31.328 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:31 smithi164 ceph-mon[24045]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T15:25:31.328 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:31.328 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:31.328 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:31.328 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:31.328 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:25:31.328 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:31.328 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:25:31.328 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:31 smithi112 ceph-mon[21780]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:31 smithi112 ceph-mon[21780]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T15:25:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:31 smithi112 ceph-mon[21780]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T15:25:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:31.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:31.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:25:31.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:31.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:25:31.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:32.258 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:32 smithi164 ceph-mon[24045]: Detected new or changed devices on smithi164 2024-04-26T15:25:32.259 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:32 smithi164 ceph-mon[24045]: Adjusting osd_memory_target on smithi164 to 15936M 2024-04-26T15:25:32.259 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:32 smithi164 ceph-mon[24045]: osdmap e36: 5 total, 5 up, 5 in 2024-04-26T15:25:32.362 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:32 smithi112 ceph-mon[21780]: Detected new or changed devices on smithi164 2024-04-26T15:25:32.363 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:32 smithi112 ceph-mon[21780]: Adjusting osd_memory_target on smithi164 to 15936M 2024-04-26T15:25:32.363 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:32 smithi112 ceph-mon[21780]: osdmap e36: 5 total, 5 up, 5 in 2024-04-26T15:25:33.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:33 smithi164 ceph-mon[24045]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:33.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:33 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:25:33.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:33 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:25:33.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:33 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:33.381 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:33 smithi112 ceph-mon[21780]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:33.381 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:33 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:25:33.381 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:33 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:25:33.381 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:33 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:34.324 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:34 smithi164 ceph-mon[24045]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:25:34.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:34 smithi112 ceph-mon[21780]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:25:35.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:35 smithi164 ceph-mon[24045]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-26T15:25:35.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:35 smithi112 ceph-mon[21780]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-26T15:25:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:36 smithi164 ceph-mon[24045]: from='client.? 172.21.15.164:0/3363282502' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5042dca7-9893-4bf1-9eac-9b01b34a2cd3"}]: dispatch 2024-04-26T15:25:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:36 smithi164 ceph-mon[24045]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5042dca7-9893-4bf1-9eac-9b01b34a2cd3"}]: dispatch 2024-04-26T15:25:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:36 smithi164 ceph-mon[24045]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5042dca7-9893-4bf1-9eac-9b01b34a2cd3"}]': finished 2024-04-26T15:25:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:36 smithi164 ceph-mon[24045]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T15:25:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:36 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:25:36.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:36 smithi164 ceph-mon[24045]: from='client.? 172.21.15.164:0/1577486698' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:25:36.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:36 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/3363282502' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5042dca7-9893-4bf1-9eac-9b01b34a2cd3"}]: dispatch 2024-04-26T15:25:36.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:36 smithi112 ceph-mon[21780]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5042dca7-9893-4bf1-9eac-9b01b34a2cd3"}]: dispatch 2024-04-26T15:25:36.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:36 smithi112 ceph-mon[21780]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5042dca7-9893-4bf1-9eac-9b01b34a2cd3"}]': finished 2024-04-26T15:25:36.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:36 smithi112 ceph-mon[21780]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T15:25:36.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:36 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:25:36.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:36 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/1577486698' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:25:37.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:37 smithi164 ceph-mon[24045]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-26T15:25:37.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:37 smithi112 ceph-mon[21780]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-26T15:25:39.335 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:39 smithi164 ceph-mon[24045]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-26T15:25:39.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:39 smithi112 ceph-mon[21780]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-26T15:25:41.345 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:41 smithi164 ceph-mon[24045]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-26T15:25:41.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:41 smithi112 ceph-mon[21780]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-26T15:25:42.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:42 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T15:25:42.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:42 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:42.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T15:25:42.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:43.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:43 smithi164 ceph-mon[24045]: Deploying daemon osd.5 on smithi164 2024-04-26T15:25:43.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:43 smithi164 ceph-mon[24045]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:25:43.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:43 smithi112 ceph-mon[21780]: Deploying daemon osd.5 on smithi164 2024-04-26T15:25:43.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:43 smithi112 ceph-mon[21780]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:25:45.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:45 smithi164 ceph-mon[24045]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:45.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:45 smithi112 ceph-mon[21780]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:47.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:47 smithi164 ceph-mon[24045]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:47.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:47 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:47.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:47 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:25:47.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:47 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:47.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:47 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:25:47.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:47 smithi112 ceph-mon[21780]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:47.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:47 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:47.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:47 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:25:47.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:47 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:47.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:47 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:25:49.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:49 smithi164 ceph-mon[24045]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:49.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:49 smithi112 ceph-mon[21780]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:50 smithi164 ceph-mon[24045]: from='osd.5 [v2:172.21.15.164:6808/1097508373,v1:172.21.15.164:6809/1097508373]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:25:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:50 smithi164 ceph-mon[24045]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:25:50.389 INFO:teuthology.orchestra.run.smithi164.stdout:Created osd(s) 5 on host 'smithi164' 2024-04-26T15:25:50.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:50 smithi112 ceph-mon[21780]: from='osd.5 [v2:172.21.15.164:6808/1097508373,v1:172.21.15.164:6809/1097508373]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:25:50.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:50 smithi112 ceph-mon[21780]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:25:50.992 DEBUG:teuthology.orchestra.run.smithi164:osd.5> sudo journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.5.service 2024-04-26T15:25:50.994 INFO:tasks.cephadm:Deploying osd.6 on smithi164 with /dev/vg_nvme/lv_2... 2024-04-26T15:25:50.994 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T15:25:51.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:51 smithi164 ceph-mon[24045]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:51.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:51 smithi164 ceph-mon[24045]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T15:25:51.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:51 smithi164 ceph-mon[24045]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T15:25:51.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:51 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:25:51.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:51 smithi164 ceph-mon[24045]: from='osd.5 [v2:172.21.15.164:6808/1097508373,v1:172.21.15.164:6809/1097508373]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:25:51.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:51 smithi164 ceph-mon[24045]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:25:51.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:51 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:51.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:51 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:51.348 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:25:51 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[31286]: 2024-04-26T15:25:51.120+0000 7f1d6837e700 -1 osd.5 0 waiting for initial osdmap 2024-04-26T15:25:51.348 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:25:51 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[31286]: 2024-04-26T15:25:51.125+0000 7f1d61512700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:25:51.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:51 smithi112 ceph-mon[21780]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:51.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:51 smithi112 ceph-mon[21780]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T15:25:51.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:51 smithi112 ceph-mon[21780]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T15:25:51.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:51 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:25:51.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:51 smithi112 ceph-mon[21780]: from='osd.5 [v2:172.21.15.164:6808/1097508373,v1:172.21.15.164:6809/1097508373]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:25:51.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:51 smithi112 ceph-mon[21780]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:25:51.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:51 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:51.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:51 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:52.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:52 smithi164 ceph-mon[24045]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-04-26T15:25:52.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:52 smithi164 ceph-mon[24045]: osdmap e39: 6 total, 5 up, 6 in 2024-04-26T15:25:52.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:52 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:25:52.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:52 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:25:52.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:52 smithi164 ceph-mon[24045]: osd.5 [v2:172.21.15.164:6808/1097508373,v1:172.21.15.164:6809/1097508373] boot 2024-04-26T15:25:52.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:52 smithi164 ceph-mon[24045]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T15:25:52.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:52 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:25:52.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:52 smithi112 ceph-mon[21780]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-04-26T15:25:52.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:52 smithi112 ceph-mon[21780]: osdmap e39: 6 total, 5 up, 6 in 2024-04-26T15:25:52.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:52 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:25:52.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:52 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:25:52.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:52 smithi112 ceph-mon[21780]: osd.5 [v2:172.21.15.164:6808/1097508373,v1:172.21.15.164:6809/1097508373] boot 2024-04-26T15:25:52.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:52 smithi112 ceph-mon[21780]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T15:25:52.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:52 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:25:53.218 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:53 smithi164 ceph-mon[24045]: purged_snaps scrub starts 2024-04-26T15:25:53.219 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:53 smithi164 ceph-mon[24045]: purged_snaps scrub ok 2024-04-26T15:25:53.219 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:53 smithi164 ceph-mon[24045]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:53.219 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:53 smithi164 ceph-mon[24045]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T15:25:53.219 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T15:25:53.219 INFO:teuthology.orchestra.run.smithi164.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T15:25:53.219 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10+0 records in 2024-04-26T15:25:53.219 INFO:teuthology.orchestra.run.smithi164.stderr:10+0 records out 2024-04-26T15:25:53.219 INFO:teuthology.orchestra.run.smithi164.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0126801 s, 827 MB/s 2024-04-26T15:25:53.219 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping successful for: 2024-04-26T15:25:53.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:53 smithi112 ceph-mon[21780]: purged_snaps scrub starts 2024-04-26T15:25:53.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:53 smithi112 ceph-mon[21780]: purged_snaps scrub ok 2024-04-26T15:25:53.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:53 smithi112 ceph-mon[21780]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:25:53.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:53 smithi112 ceph-mon[21780]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T15:25:55.061 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi164:vg_nvme/lv_2 2024-04-26T15:25:55.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:55 smithi164 ceph-mon[24045]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:25:55.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:55 smithi164 ceph-mon[24045]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T15:25:55.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:55 smithi112 ceph-mon[21780]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:25:55.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:55 smithi112 ceph-mon[21780]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T15:25:56.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:56 smithi164 ceph-mon[24045]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T15:25:56.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:56 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:56.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:56 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:56.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:56 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:56.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:56 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:56.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:56 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:56.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:56 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:25:56.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:56 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:56.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:56 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:25:56.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:56 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:56.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:56 smithi112 ceph-mon[21780]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T15:25:56.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:56 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:56.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:56 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:56.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:56 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:25:56.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:56 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:56.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:56 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:56.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:56 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:25:56.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:56 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:56.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:56 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:25:56.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:56 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:25:57.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:57 smithi164 ceph-mon[24045]: Detected new or changed devices on smithi164 2024-04-26T15:25:57.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:57 smithi164 ceph-mon[24045]: Adjusting osd_memory_target on smithi164 to 7968M 2024-04-26T15:25:57.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:57 smithi164 ceph-mon[24045]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:25:57.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:57 smithi164 ceph-mon[24045]: osdmap e44: 6 total, 6 up, 6 in 2024-04-26T15:25:57.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:57 smithi112 ceph-mon[21780]: Detected new or changed devices on smithi164 2024-04-26T15:25:57.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:57 smithi112 ceph-mon[21780]: Adjusting osd_memory_target on smithi164 to 7968M 2024-04-26T15:25:57.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:57 smithi112 ceph-mon[21780]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:25:57.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:57 smithi112 ceph-mon[21780]: osdmap e44: 6 total, 6 up, 6 in 2024-04-26T15:25:58.343 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:58 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:25:58.343 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:58 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:25:58.343 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:58 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:58.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:58 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:25:58.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:58 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:25:58.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:58 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:25:59.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:59 smithi164 ceph-mon[24045]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:25:59.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:25:59 smithi164 ceph-mon[24045]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:25:59.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:59 smithi112 ceph-mon[21780]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:25:59.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:25:59 smithi112 ceph-mon[21780]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:00.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:00 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/257339380' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e80c2ba-9b4b-4eef-a2ab-144c5ec8555c"}]: dispatch 2024-04-26T15:26:00.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:00 smithi112 ceph-mon[21780]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e80c2ba-9b4b-4eef-a2ab-144c5ec8555c"}]: dispatch 2024-04-26T15:26:00.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:00 smithi112 ceph-mon[21780]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0e80c2ba-9b4b-4eef-a2ab-144c5ec8555c"}]': finished 2024-04-26T15:26:00.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:00 smithi112 ceph-mon[21780]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T15:26:00.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:26:00.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:00 smithi164 ceph-mon[24045]: from='client.? 172.21.15.164:0/257339380' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e80c2ba-9b4b-4eef-a2ab-144c5ec8555c"}]: dispatch 2024-04-26T15:26:00.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:00 smithi164 ceph-mon[24045]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e80c2ba-9b4b-4eef-a2ab-144c5ec8555c"}]: dispatch 2024-04-26T15:26:00.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:00 smithi164 ceph-mon[24045]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0e80c2ba-9b4b-4eef-a2ab-144c5ec8555c"}]': finished 2024-04-26T15:26:00.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:00 smithi164 ceph-mon[24045]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T15:26:00.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:26:01.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:01 smithi112 ceph-mon[21780]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:01.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:01 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/1046994821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:26:01.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:01 smithi164 ceph-mon[24045]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:01.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:01 smithi164 ceph-mon[24045]: from='client.? 172.21.15.164:0/1046994821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:26:03.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:03 smithi112 ceph-mon[21780]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:03.562 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:03 smithi164 ceph-mon[24045]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:05.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:05 smithi164 ceph-mon[24045]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:05.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:05 smithi112 ceph-mon[21780]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:06.795 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:06 smithi164 ceph-mon[24045]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:06.795 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:06 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T15:26:06.796 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:06 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:06.796 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:06 smithi164 ceph-mon[24045]: Deploying daemon osd.6 on smithi164 2024-04-26T15:26:07.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:06 smithi112 ceph-mon[21780]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:07.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:06 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T15:26:07.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:06 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:07.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:06 smithi112 ceph-mon[21780]: Deploying daemon osd.6 on smithi164 2024-04-26T15:26:08.998 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:08 smithi164 ceph-mon[24045]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:09.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:08 smithi112 ceph-mon[21780]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:10.774 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:10 smithi164 ceph-mon[24045]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:11.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:10 smithi112 ceph-mon[21780]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:12.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:12 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:12.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:12 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:26:12.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:12 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:12.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:12 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:26:12.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:12 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:12.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:12 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:26:12.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:12 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:12.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:12 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:26:13.404 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:13 smithi164 ceph-mon[24045]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:13.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:13 smithi112 ceph-mon[21780]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:15.246 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:15 smithi164 ceph-mon[24045]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:15.246 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:15 smithi164 ceph-mon[24045]: from='osd.6 [v2:172.21.15.164:6816/3245318540,v1:172.21.15.164:6817/3245318540]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:26:15.246 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:15 smithi164 ceph-mon[24045]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:26:15.247 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:15 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:15.247 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:15 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:15.278 INFO:teuthology.orchestra.run.smithi164.stdout:Created osd(s) 6 on host 'smithi164' 2024-04-26T15:26:15.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:15 smithi112 ceph-mon[21780]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:15.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:15 smithi112 ceph-mon[21780]: from='osd.6 [v2:172.21.15.164:6816/3245318540,v1:172.21.15.164:6817/3245318540]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:26:15.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:15 smithi112 ceph-mon[21780]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:26:15.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:15 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:15.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:15 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:15.914 DEBUG:teuthology.orchestra.run.smithi164:osd.6> sudo journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.6.service 2024-04-26T15:26:15.916 INFO:tasks.cephadm:Deploying osd.7 on smithi164 with /dev/vg_nvme/lv_1... 2024-04-26T15:26:15.916 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T15:26:16.453 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:16 smithi164 ceph-mon[24045]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T15:26:16.453 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:16 smithi164 ceph-mon[24045]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T15:26:16.453 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:16 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:26:16.453 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:16 smithi164 ceph-mon[24045]: from='osd.6 [v2:172.21.15.164:6816/3245318540,v1:172.21.15.164:6817/3245318540]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:26:16.453 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:16 smithi164 ceph-mon[24045]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:26:16.453 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:16 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:16.453 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:16 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:26:16.453 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:16 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:16.454 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:16 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:26:16.454 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:26:16 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[34280]: 2024-04-26T15:26:16.173+0000 7fa646e3d700 -1 osd.6 0 waiting for initial osdmap 2024-04-26T15:26:16.454 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:26:16 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[34280]: 2024-04-26T15:26:16.180+0000 7fa6417d4700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:26:16.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:16 smithi112 ceph-mon[21780]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T15:26:16.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:16 smithi112 ceph-mon[21780]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T15:26:16.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:16 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:26:16.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:16 smithi112 ceph-mon[21780]: from='osd.6 [v2:172.21.15.164:6816/3245318540,v1:172.21.15.164:6817/3245318540]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:26:16.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:16 smithi112 ceph-mon[21780]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:26:16.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:16 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:16.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:16 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:26:16.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:16 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:16.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:16 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:26:17.430 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:17 smithi164 ceph-mon[24045]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:17.431 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:17 smithi164 ceph-mon[24045]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-04-26T15:26:17.431 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:17 smithi164 ceph-mon[24045]: osdmap e47: 7 total, 6 up, 7 in 2024-04-26T15:26:17.431 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:17 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:26:17.431 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:17 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:26:17.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:17 smithi112 ceph-mon[21780]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:17.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:17 smithi112 ceph-mon[21780]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-04-26T15:26:17.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:17 smithi112 ceph-mon[21780]: osdmap e47: 7 total, 6 up, 7 in 2024-04-26T15:26:17.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:17 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:26:17.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:17 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:26:18.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:18 smithi164 ceph-mon[24045]: purged_snaps scrub starts 2024-04-26T15:26:18.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:18 smithi164 ceph-mon[24045]: purged_snaps scrub ok 2024-04-26T15:26:18.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:18 smithi164 ceph-mon[24045]: osd.6 [v2:172.21.15.164:6816/3245318540,v1:172.21.15.164:6817/3245318540] boot 2024-04-26T15:26:18.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:18 smithi164 ceph-mon[24045]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T15:26:18.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:18 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:26:18.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:18 smithi164 ceph-mon[24045]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T15:26:18.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:18 smithi112 ceph-mon[21780]: purged_snaps scrub starts 2024-04-26T15:26:18.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:18 smithi112 ceph-mon[21780]: purged_snaps scrub ok 2024-04-26T15:26:18.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:18 smithi112 ceph-mon[21780]: osd.6 [v2:172.21.15.164:6816/3245318540,v1:172.21.15.164:6817/3245318540] boot 2024-04-26T15:26:18.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:18 smithi112 ceph-mon[21780]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T15:26:18.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:18 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:26:18.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:18 smithi112 ceph-mon[21780]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T15:26:19.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:19 smithi112 ceph-mon[21780]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:19.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:19 smithi112 ceph-mon[21780]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T15:26:19.499 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:19 smithi164 ceph-mon[24045]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:26:19.499 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:19 smithi164 ceph-mon[24045]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T15:26:19.580 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T15:26:19.580 INFO:teuthology.orchestra.run.smithi164.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T15:26:19.580 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10+0 records in 2024-04-26T15:26:19.580 INFO:teuthology.orchestra.run.smithi164.stderr:10+0 records out 2024-04-26T15:26:19.581 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0129172 s, 812 MB/s 2024-04-26T15:26:19.581 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping successful for: 2024-04-26T15:26:20.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi164:vg_nvme/lv_1 2024-04-26T15:26:21.030 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:20 smithi164 ceph-mon[24045]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:21.031 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:20 smithi164 ceph-mon[24045]: Detected new or changed devices on smithi164 2024-04-26T15:26:21.031 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:21.031 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:21.031 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:21.031 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:21.031 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:20 smithi164 ceph-mon[24045]: Adjusting osd_memory_target on smithi164 to 5312M 2024-04-26T15:26:21.031 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:21.031 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:21.031 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:20 smithi164 ceph-mon[24045]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T15:26:21.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:20 smithi112 ceph-mon[21780]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:21.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:20 smithi112 ceph-mon[21780]: Detected new or changed devices on smithi164 2024-04-26T15:26:21.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:21.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:21.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:21.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:21.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:20 smithi112 ceph-mon[21780]: Adjusting osd_memory_target on smithi164 to 5312M 2024-04-26T15:26:21.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:21.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:21.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:20 smithi112 ceph-mon[21780]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T15:26:22.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:22 smithi164 ceph-mon[24045]: osdmap e52: 7 total, 7 up, 7 in 2024-04-26T15:26:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:22 smithi112 ceph-mon[21780]: osdmap e52: 7 total, 7 up, 7 in 2024-04-26T15:26:23.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:23 smithi164 ceph-mon[24045]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:23.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:23 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:26:23.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:23 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:26:23.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:23 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:23.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:23 smithi112 ceph-mon[21780]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:23.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:23 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:26:23.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:23 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:26:23.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:23 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:24.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:24 smithi164 ceph-mon[24045]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:26:24.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:24 smithi112 ceph-mon[21780]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:26:25.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:25 smithi112 ceph-mon[21780]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-26T15:26:25.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:25 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/2209842489' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ec486d6f-2acf-4175-9ed2-5fbe142944c7"}]: dispatch 2024-04-26T15:26:25.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:25 smithi112 ceph-mon[21780]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ec486d6f-2acf-4175-9ed2-5fbe142944c7"}]: dispatch 2024-04-26T15:26:25.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:25 smithi112 ceph-mon[21780]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ec486d6f-2acf-4175-9ed2-5fbe142944c7"}]': finished 2024-04-26T15:26:25.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:25 smithi112 ceph-mon[21780]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T15:26:25.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:25 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:26:25.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:25 smithi164 ceph-mon[24045]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-26T15:26:25.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:25 smithi164 ceph-mon[24045]: from='client.? 172.21.15.164:0/2209842489' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ec486d6f-2acf-4175-9ed2-5fbe142944c7"}]: dispatch 2024-04-26T15:26:25.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:25 smithi164 ceph-mon[24045]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ec486d6f-2acf-4175-9ed2-5fbe142944c7"}]: dispatch 2024-04-26T15:26:25.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:25 smithi164 ceph-mon[24045]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ec486d6f-2acf-4175-9ed2-5fbe142944c7"}]': finished 2024-04-26T15:26:25.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:25 smithi164 ceph-mon[24045]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T15:26:25.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:25 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:26:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:26 smithi164 ceph-mon[24045]: from='client.? 172.21.15.164:0/3669321072' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:26:26.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:26 smithi112 ceph-mon[21780]: from='client.? 172.21.15.164:0/3669321072' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:26:27.523 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:27 smithi164 ceph-mon[24045]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:26:27.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:27 smithi112 ceph-mon[21780]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:26:28.519 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:26:28.519 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:26:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:26:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:26:29.516 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:29 smithi164 ceph-mon[24045]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-26T15:26:29.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:29 smithi112 ceph-mon[21780]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-26T15:26:31.509 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:31 smithi164 ceph-mon[24045]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-26T15:26:31.509 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T15:26:31.509 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:31.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:31 smithi112 ceph-mon[21780]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-26T15:26:31.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T15:26:31.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:32.321 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:32 smithi164 ceph-mon[24045]: Deploying daemon osd.7 on smithi164 2024-04-26T15:26:32.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:32 smithi112 ceph-mon[21780]: Deploying daemon osd.7 on smithi164 2024-04-26T15:26:33.535 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:33 smithi164 ceph-mon[24045]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:26:33.612 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:33 smithi112 ceph-mon[21780]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:26:35.427 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:35 smithi164 ceph-mon[24045]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:35.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:35 smithi112 ceph-mon[21780]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:37.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:37 smithi164 ceph-mon[24045]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:37.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:37 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:37.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:37 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:26:37.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:37 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:37.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:37 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:26:37.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:37 smithi112 ceph-mon[21780]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:37.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:37 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:37.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:37 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:26:37.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:37 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:37.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:37 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:26:39.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:39 smithi164 ceph-mon[24045]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:39.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:39 smithi112 ceph-mon[21780]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:39.681 INFO:teuthology.orchestra.run.smithi164.stdout:Created osd(s) 7 on host 'smithi164' 2024-04-26T15:26:40.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:40 smithi164 ceph-mon[24045]: from='osd.7 [v2:172.21.15.164:6824/897796359,v1:172.21.15.164:6825/897796359]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:26:40.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:40 smithi164 ceph-mon[24045]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:26:40.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:40 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:40.292 DEBUG:teuthology.orchestra.run.smithi164:osd.7> sudo journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.7.service 2024-04-26T15:26:40.294 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-26T15:26:40.295 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-26T15:26:40.320 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:40 smithi112 ceph-mon[21780]: from='osd.7 [v2:172.21.15.164:6824/897796359,v1:172.21.15.164:6825/897796359]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:26:40.320 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:40 smithi112 ceph-mon[21780]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:26:40.320 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:40 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:41.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:41 smithi164 ceph-mon[24045]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:41.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:41 smithi164 ceph-mon[24045]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T15:26:41.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:41 smithi164 ceph-mon[24045]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T15:26:41.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:41 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:26:41.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:41 smithi164 ceph-mon[24045]: from='osd.7 [v2:172.21.15.164:6824/897796359,v1:172.21.15.164:6825/897796359]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:26:41.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:41 smithi164 ceph-mon[24045]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:26:41.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:41 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:41.098 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:26:41 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[37344]: 2024-04-26T15:26:41.070+0000 7f28e655c700 -1 osd.7 0 waiting for initial osdmap 2024-04-26T15:26:41.099 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:26:41 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[37344]: 2024-04-26T15:26:41.079+0000 7f28e26f6700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:26:41.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:41 smithi112 ceph-mon[21780]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:41.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:41 smithi112 ceph-mon[21780]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T15:26:41.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:41 smithi112 ceph-mon[21780]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T15:26:41.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:41 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:26:41.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:41 smithi112 ceph-mon[21780]: from='osd.7 [v2:172.21.15.164:6824/897796359,v1:172.21.15.164:6825/897796359]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:26:41.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:41 smithi112 ceph-mon[21780]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:26:41.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:41 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:42.143 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:42 smithi164 ceph-mon[24045]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-04-26T15:26:42.143 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:42 smithi164 ceph-mon[24045]: osdmap e55: 8 total, 7 up, 8 in 2024-04-26T15:26:42.143 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:42 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:26:42.143 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:42 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:26:42.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:42 smithi112 ceph-mon[21780]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-04-26T15:26:42.269 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:42 smithi112 ceph-mon[21780]: osdmap e55: 8 total, 7 up, 8 in 2024-04-26T15:26:42.269 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:26:42.269 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:42 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:26:42.340 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:26:43.035 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1714145202,"num_in_osds":8,"osd_in_since":1714145184,"num_remapped_pgs":1} 2024-04-26T15:26:43.035 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T15:26:43.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:43 smithi164 ceph-mon[24045]: purged_snaps scrub starts 2024-04-26T15:26:43.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:43 smithi164 ceph-mon[24045]: purged_snaps scrub ok 2024-04-26T15:26:43.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:43 smithi164 ceph-mon[24045]: pgmap v170: 1 pgs: 1 peering; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:43.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:43 smithi164 ceph-mon[24045]: osd.7 [v2:172.21.15.164:6824/897796359,v1:172.21.15.164:6825/897796359] boot 2024-04-26T15:26:43.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:43 smithi164 ceph-mon[24045]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T15:26:43.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:43 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:26:43.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:43 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/3202452430' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T15:26:43.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:43 smithi164 ceph-mon[24045]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T15:26:43.291 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:26:43.373 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:43 smithi112 ceph-mon[21780]: purged_snaps scrub starts 2024-04-26T15:26:43.373 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:43 smithi112 ceph-mon[21780]: purged_snaps scrub ok 2024-04-26T15:26:43.373 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:43 smithi112 ceph-mon[21780]: pgmap v170: 1 pgs: 1 peering; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:26:43.373 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:43 smithi112 ceph-mon[21780]: osd.7 [v2:172.21.15.164:6824/897796359,v1:172.21.15.164:6825/897796359] boot 2024-04-26T15:26:43.373 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:43 smithi112 ceph-mon[21780]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T15:26:43.373 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:43 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:26:43.373 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:43 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/3202452430' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T15:26:43.373 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:43 smithi112 ceph-mon[21780]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T15:26:44.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: pgmap v173: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:26:44.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: Detected new or changed devices on smithi164 2024-04-26T15:26:44.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:44.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:44.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:44.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:44.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:44.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: Adjusting osd_memory_target on smithi164 to 3984M 2024-04-26T15:26:44.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:44.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:44.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:26:44.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:44.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:26:44.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:44.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:44 smithi112 ceph-mon[21780]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T15:26:45.082 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:26:45.082 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":58,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","created":"2024-04-26T15:20:40.883831+0000","modified":"2024-04-26T15:26:44.072277+0000","last_up_change":"2024-04-26T15:26:42.065033+0000","last_in_change":"2024-04-26T15:26:24.693410+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T15:24:37.984143+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"57f44c85-dc23-45ed-909d-bfaa1b8bbbec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":2249725715},{"type":"v1","addr":"172.21.15.112:6803","nonce":2249725715}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":2249725715},{"type":"v1","addr":"172.21.15.112:6805","nonce":2249725715}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":2249725715},{"type":"v1","addr":"172.21.15.112:6809","nonce":2249725715}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":2249725715},{"type":"v1","addr":"172.21.15.112:6807","nonce":2249725715}]},"public_addr":"172.21.15.112:6803/2249725715","cluster_addr":"172.21.15.112:6805/2249725715","heartbeat_back_addr":"172.21.15.112:6809/2249725715","heartbeat_front_addr":"172.21.15.112:6807/2249725715","state":["exists","up"]},{"osd":1,"uuid":"baa30761-660d-452a-bdfe-00569457e9a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":20141175},{"type":"v1","addr":"172.21.15.112:6811","nonce":20141175}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":20141175},{"type":"v1","addr":"172.21.15.112:6813","nonce":20141175}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":20141175},{"type":"v1","addr":"172.21.15.112:6817","nonce":20141175}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":20141175},{"type":"v1","addr":"172.21.15.112:6815","nonce":20141175}]},"public_addr":"172.21.15.112:6811/20141175","cluster_addr":"172.21.15.112:6813/20141175","heartbeat_back_addr":"172.21.15.112:6817/20141175","heartbeat_front_addr":"172.21.15.112:6815/20141175","state":["exists","up"]},{"osd":2,"uuid":"17e98a3f-1edf-4186-8a34-2dce3e86d234","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":1768051982},{"type":"v1","addr":"172.21.15.112:6819","nonce":1768051982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":1768051982},{"type":"v1","addr":"172.21.15.112:6821","nonce":1768051982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":1768051982},{"type":"v1","addr":"172.21.15.112:6825","nonce":1768051982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":1768051982},{"type":"v1","addr":"172.21.15.112:6823","nonce":1768051982}]},"public_addr":"172.21.15.112:6819/1768051982","cluster_addr":"172.21.15.112:6821/1768051982","heartbeat_back_addr":"172.21.15.112:6825/1768051982","heartbeat_front_addr":"172.21.15.112:6823/1768051982","state":["exists","up"]},{"osd":3,"uuid":"6c85a093-d36e-4c61-bd75-95c2f09be643","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":1408970703},{"type":"v1","addr":"172.21.15.112:6827","nonce":1408970703}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":1408970703},{"type":"v1","addr":"172.21.15.112:6829","nonce":1408970703}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":1408970703},{"type":"v1","addr":"172.21.15.112:6833","nonce":1408970703}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":1408970703},{"type":"v1","addr":"172.21.15.112:6831","nonce":1408970703}]},"public_addr":"172.21.15.112:6827/1408970703","cluster_addr":"172.21.15.112:6829/1408970703","heartbeat_back_addr":"172.21.15.112:6833/1408970703","heartbeat_front_addr":"172.21.15.112:6831/1408970703","state":["exists","up"]},{"osd":4,"uuid":"31d857fe-771a-474b-8faf-bd38342cf00f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":3452220802},{"type":"v1","addr":"172.21.15.164:6801","nonce":3452220802}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":3452220802},{"type":"v1","addr":"172.21.15.164:6803","nonce":3452220802}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":3452220802},{"type":"v1","addr":"172.21.15.164:6807","nonce":3452220802}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":3452220802},{"type":"v1","addr":"172.21.15.164:6805","nonce":3452220802}]},"public_addr":"172.21.15.164:6801/3452220802","cluster_addr":"172.21.15.164:6803/3452220802","heartbeat_back_addr":"172.21.15.164:6807/3452220802","heartbeat_front_addr":"172.21.15.164:6805/3452220802","state":["exists","up"]},{"osd":5,"uuid":"5042dca7-9893-4bf1-9eac-9b01b34a2cd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":1097508373},{"type":"v1","addr":"172.21.15.164:6809","nonce":1097508373}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":1097508373},{"type":"v1","addr":"172.21.15.164:6811","nonce":1097508373}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":1097508373},{"type":"v1","addr":"172.21.15.164:6815","nonce":1097508373}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":1097508373},{"type":"v1","addr":"172.21.15.164:6813","nonce":1097508373}]},"public_addr":"172.21.15.164:6809/1097508373","cluster_addr":"172.21.15.164:6811/1097508373","heartbeat_back_addr":"172.21.15.164:6815/1097508373","heartbeat_front_addr":"172.21.15.164:6813/1097508373","state":["exists","up"]},{"osd":6,"uuid":"0e80c2ba-9b4b-4eef-a2ab-144c5ec8555c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":3245318540},{"type":"v1","addr":"172.21.15.164:6817","nonce":3245318540}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":3245318540},{"type":"v1","addr":"172.21.15.164:6819","nonce":3245318540}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":3245318540},{"type":"v1","addr":"172.21.15.164:6823","nonce":3245318540}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":3245318540},{"type":"v1","addr":"172.21.15.164:6821","nonce":3245318540}]},"public_addr":"172.21.15.164:6817/3245318540","cluster_addr":"172.21.15.164:6819/3245318540","heartbeat_back_addr":"172.21.15.164:6823/3245318540","heartbeat_front_addr":"172.21.15.164:6821/3245318540","state":["exists","up"]},{"osd":7,"uuid":"ec486d6f-2acf-4175-9ed2-5fbe142944c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":897796359},{"type":"v1","addr":"172.21.15.164:6825","nonce":897796359}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":897796359},{"type":"v1","addr":"172.21.15.164:6827","nonce":897796359}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":897796359},{"type":"v1","addr":"172.21.15.164:6831","nonce":897796359}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":897796359},{"type":"v1","addr":"172.21.15.164:6829","nonce":897796359}]},"public_addr":"172.21.15.164:6825/897796359","cluster_addr":"172.21.15.164:6827/897796359","heartbeat_back_addr":"172.21.15.164:6831/897796359","heartbeat_front_addr":"172.21.15.164:6829/897796359","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:23:33.156128+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:24:04.246089+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:24:34.348238+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:25:03.565446+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:25:26.088109+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:25:50.475116+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:26:15.272437+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:26:40.172061+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[{"pgid":"1.0","osds":[0,6]}],"primary_temp":[],"blocklist":{"172.21.15.112:6801/3398416842":"2024-04-27T15:22:27.152936+0000","172.21.15.112:0/447240260":"2024-04-27T15:22:27.152936+0000","172.21.15.112:0/1201901815":"2024-04-27T15:21:36.095747+0000","172.21.15.112:6800/3398416842":"2024-04-27T15:22:27.152936+0000","172.21.15.112:0/953234053":"2024-04-27T15:21:36.095747+0000","172.21.15.112:0/3385065922":"2024-04-27T15:22:27.152936+0000","172.21.15.112:0/4214048146":"2024-04-27T15:21:36.095747+0000","172.21.15.112:6801/1647678753":"2024-04-27T15:21:36.095747+0000","172.21.15.112:6800/1647678753":"2024-04-27T15:21:36.095747+0000","172.21.15.112:0/2379216906":"2024-04-27T15:21:06.648104+0000","172.21.15.112:0/3306085235":"2024-04-27T15:22:27.152936+0000","172.21.15.112:6801/478922692":"2024-04-27T15:21:06.648104+0000","172.21.15.112:0/3064986537":"2024-04-27T15:21:06.648104+0000","172.21.15.112:6800/478922692":"2024-04-27T15:21:06.648104+0000","172.21.15.112:0/2069494748":"2024-04-27T15:21:06.648104+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T15:26:45.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: pgmap v173: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:26:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: Detected new or changed devices on smithi164 2024-04-26T15:26:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:26:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: Adjusting osd_memory_target on smithi164 to 3984M 2024-04-26T15:26:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:26:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:26:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:26:45.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:26:45.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:44 smithi164 ceph-mon[24045]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T15:26:45.847 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-26T15:24:37.984143+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-26T15:26:45.847 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-26T15:26:46.060 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:26:46.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:45 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/3073574528' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:26:46.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:45 smithi164 ceph-mon[24045]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T15:26:46.182 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:45 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/3073574528' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:26:46.182 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:45 smithi112 ceph-mon[21780]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T15:26:47.222 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:47 smithi112 ceph-mon[21780]: pgmap v176: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:26:47.222 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:47 smithi112 ceph-mon[21780]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T15:26:47.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:47 smithi164 ceph-mon[24045]: pgmap v176: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:26:47.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:47 smithi164 ceph-mon[24045]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T15:26:47.794 INFO:teuthology.orchestra.run.smithi112.stdout:pg_num: 1 2024-04-26T15:26:48.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:48 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/2980266603' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T15:26:48.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:48 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2980266603' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T15:26:48.539 INFO:tasks.cephadm:Setting up client nodes... 2024-04-26T15:26:48.540 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T15:26:49.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:49 smithi164 ceph-mon[24045]: pgmap v178: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:26:49.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:49 smithi112 ceph-mon[21780]: pgmap v178: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:26:50.503 INFO:teuthology.orchestra.run.smithi112.stdout:[client.0] 2024-04-26T15:26:50.503 INFO:teuthology.orchestra.run.smithi112.stdout: key = AQC6xytm6XHUHRAAbYBicKH0txzJUH3tcYUxLw== 2024-04-26T15:26:51.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:51 smithi112 ceph-mon[21780]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:26:51.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:51 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1384606319' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:26:51.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:51 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1384606319' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T15:26:51.236 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-26T15:26:51.237 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-26T15:26:51.237 DEBUG:teuthology.orchestra.run.smithi112:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-26T15:26:51.271 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T15:26:51.271 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T15:26:51.272 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T15:26:51.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:51 smithi164 ceph-mon[24045]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:26:51.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:51 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1384606319' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:26:51.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:51 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1384606319' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T15:26:51.519 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:26:53.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:53 smithi164 ceph-mon[24045]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-26T15:26:53.392 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:26:53.392 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:53 smithi112 ceph-mon[21780]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-26T15:26:54.157 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi112.llioug","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":949107427},{"type":"v1","addr":"172.21.15.112:6801","nonce":949107427}]},"active_addr":"172.21.15.112:6801/949107427","active_change":"2024-04-26T15:22:27.153346+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14224,"name":"smithi164.tuhwus","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.112:8443/","prometheus":"http://172.21.15.112:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.112:0","nonce":3181303815}]},{"addrvec":[{"type":"v2","addr":"172.21.15.112:0","nonce":637780854}]},{"addrvec":[{"type":"v2","addr":"172.21.15.112:0","nonce":2123084188}]},{"addrvec":[{"type":"v2","addr":"172.21.15.112:0","nonce":1559671544}]}]}} 2024-04-26T15:26:54.162 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T15:26:54.163 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T15:26:54.163 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T15:26:54.370 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:26:54.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:54 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/39799724' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T15:26:54.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:54 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/39799724' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T15:26:55.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:55 smithi112 ceph-mon[21780]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-26T15:26:55.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:55 smithi164 ceph-mon[24045]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-26T15:26:56.204 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:26:56.204 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":60,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","created":"2024-04-26T15:20:40.883831+0000","modified":"2024-04-26T15:26:46.083342+0000","last_up_change":"2024-04-26T15:26:42.065033+0000","last_in_change":"2024-04-26T15:26:24.693410+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T15:24:37.984143+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"57f44c85-dc23-45ed-909d-bfaa1b8bbbec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":2249725715},{"type":"v1","addr":"172.21.15.112:6803","nonce":2249725715}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":2249725715},{"type":"v1","addr":"172.21.15.112:6805","nonce":2249725715}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":2249725715},{"type":"v1","addr":"172.21.15.112:6809","nonce":2249725715}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":2249725715},{"type":"v1","addr":"172.21.15.112:6807","nonce":2249725715}]},"public_addr":"172.21.15.112:6803/2249725715","cluster_addr":"172.21.15.112:6805/2249725715","heartbeat_back_addr":"172.21.15.112:6809/2249725715","heartbeat_front_addr":"172.21.15.112:6807/2249725715","state":["exists","up"]},{"osd":1,"uuid":"baa30761-660d-452a-bdfe-00569457e9a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":20141175},{"type":"v1","addr":"172.21.15.112:6811","nonce":20141175}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":20141175},{"type":"v1","addr":"172.21.15.112:6813","nonce":20141175}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":20141175},{"type":"v1","addr":"172.21.15.112:6817","nonce":20141175}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":20141175},{"type":"v1","addr":"172.21.15.112:6815","nonce":20141175}]},"public_addr":"172.21.15.112:6811/20141175","cluster_addr":"172.21.15.112:6813/20141175","heartbeat_back_addr":"172.21.15.112:6817/20141175","heartbeat_front_addr":"172.21.15.112:6815/20141175","state":["exists","up"]},{"osd":2,"uuid":"17e98a3f-1edf-4186-8a34-2dce3e86d234","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":1768051982},{"type":"v1","addr":"172.21.15.112:6819","nonce":1768051982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":1768051982},{"type":"v1","addr":"172.21.15.112:6821","nonce":1768051982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":1768051982},{"type":"v1","addr":"172.21.15.112:6825","nonce":1768051982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":1768051982},{"type":"v1","addr":"172.21.15.112:6823","nonce":1768051982}]},"public_addr":"172.21.15.112:6819/1768051982","cluster_addr":"172.21.15.112:6821/1768051982","heartbeat_back_addr":"172.21.15.112:6825/1768051982","heartbeat_front_addr":"172.21.15.112:6823/1768051982","state":["exists","up"]},{"osd":3,"uuid":"6c85a093-d36e-4c61-bd75-95c2f09be643","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":1408970703},{"type":"v1","addr":"172.21.15.112:6827","nonce":1408970703}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":1408970703},{"type":"v1","addr":"172.21.15.112:6829","nonce":1408970703}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":1408970703},{"type":"v1","addr":"172.21.15.112:6833","nonce":1408970703}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":1408970703},{"type":"v1","addr":"172.21.15.112:6831","nonce":1408970703}]},"public_addr":"172.21.15.112:6827/1408970703","cluster_addr":"172.21.15.112:6829/1408970703","heartbeat_back_addr":"172.21.15.112:6833/1408970703","heartbeat_front_addr":"172.21.15.112:6831/1408970703","state":["exists","up"]},{"osd":4,"uuid":"31d857fe-771a-474b-8faf-bd38342cf00f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":3452220802},{"type":"v1","addr":"172.21.15.164:6801","nonce":3452220802}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":3452220802},{"type":"v1","addr":"172.21.15.164:6803","nonce":3452220802}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":3452220802},{"type":"v1","addr":"172.21.15.164:6807","nonce":3452220802}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":3452220802},{"type":"v1","addr":"172.21.15.164:6805","nonce":3452220802}]},"public_addr":"172.21.15.164:6801/3452220802","cluster_addr":"172.21.15.164:6803/3452220802","heartbeat_back_addr":"172.21.15.164:6807/3452220802","heartbeat_front_addr":"172.21.15.164:6805/3452220802","state":["exists","up"]},{"osd":5,"uuid":"5042dca7-9893-4bf1-9eac-9b01b34a2cd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":1097508373},{"type":"v1","addr":"172.21.15.164:6809","nonce":1097508373}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":1097508373},{"type":"v1","addr":"172.21.15.164:6811","nonce":1097508373}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":1097508373},{"type":"v1","addr":"172.21.15.164:6815","nonce":1097508373}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":1097508373},{"type":"v1","addr":"172.21.15.164:6813","nonce":1097508373}]},"public_addr":"172.21.15.164:6809/1097508373","cluster_addr":"172.21.15.164:6811/1097508373","heartbeat_back_addr":"172.21.15.164:6815/1097508373","heartbeat_front_addr":"172.21.15.164:6813/1097508373","state":["exists","up"]},{"osd":6,"uuid":"0e80c2ba-9b4b-4eef-a2ab-144c5ec8555c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":3245318540},{"type":"v1","addr":"172.21.15.164:6817","nonce":3245318540}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":3245318540},{"type":"v1","addr":"172.21.15.164:6819","nonce":3245318540}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":3245318540},{"type":"v1","addr":"172.21.15.164:6823","nonce":3245318540}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":3245318540},{"type":"v1","addr":"172.21.15.164:6821","nonce":3245318540}]},"public_addr":"172.21.15.164:6817/3245318540","cluster_addr":"172.21.15.164:6819/3245318540","heartbeat_back_addr":"172.21.15.164:6823/3245318540","heartbeat_front_addr":"172.21.15.164:6821/3245318540","state":["exists","up"]},{"osd":7,"uuid":"ec486d6f-2acf-4175-9ed2-5fbe142944c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":897796359},{"type":"v1","addr":"172.21.15.164:6825","nonce":897796359}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":897796359},{"type":"v1","addr":"172.21.15.164:6827","nonce":897796359}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":897796359},{"type":"v1","addr":"172.21.15.164:6831","nonce":897796359}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":897796359},{"type":"v1","addr":"172.21.15.164:6829","nonce":897796359}]},"public_addr":"172.21.15.164:6825/897796359","cluster_addr":"172.21.15.164:6827/897796359","heartbeat_back_addr":"172.21.15.164:6831/897796359","heartbeat_front_addr":"172.21.15.164:6829/897796359","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:23:33.156128+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:24:04.246089+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:24:34.348238+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:25:03.565446+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:25:26.088109+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:25:50.475116+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:26:15.272437+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:26:40.172061+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.112:6801/3398416842":"2024-04-27T15:22:27.152936+0000","172.21.15.112:0/447240260":"2024-04-27T15:22:27.152936+0000","172.21.15.112:0/1201901815":"2024-04-27T15:21:36.095747+0000","172.21.15.112:6800/3398416842":"2024-04-27T15:22:27.152936+0000","172.21.15.112:0/953234053":"2024-04-27T15:21:36.095747+0000","172.21.15.112:0/3385065922":"2024-04-27T15:22:27.152936+0000","172.21.15.112:0/4214048146":"2024-04-27T15:21:36.095747+0000","172.21.15.112:6801/1647678753":"2024-04-27T15:21:36.095747+0000","172.21.15.112:6800/1647678753":"2024-04-27T15:21:36.095747+0000","172.21.15.112:0/2379216906":"2024-04-27T15:21:06.648104+0000","172.21.15.112:0/3306085235":"2024-04-27T15:22:27.152936+0000","172.21.15.112:6801/478922692":"2024-04-27T15:21:06.648104+0000","172.21.15.112:0/3064986537":"2024-04-27T15:21:06.648104+0000","172.21.15.112:6800/478922692":"2024-04-27T15:21:06.648104+0000","172.21.15.112:0/2069494748":"2024-04-27T15:21:06.648104+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T15:26:56.889 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T15:26:56.890 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T15:26:57.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:56 smithi164 ceph-mon[24045]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:26:57.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:56 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/3715522925' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:26:57.100 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:26:57.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:56 smithi112 ceph-mon[21780]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:26:57.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:56 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/3715522925' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:26:58.958 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:26:58.958 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":60,"fsid":"71e28ad8-03e0-11ef-bc93-c7b262605968","created":"2024-04-26T15:20:40.883831+0000","modified":"2024-04-26T15:26:46.083342+0000","last_up_change":"2024-04-26T15:26:42.065033+0000","last_in_change":"2024-04-26T15:26:24.693410+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T15:24:37.984143+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"57f44c85-dc23-45ed-909d-bfaa1b8bbbec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":2249725715},{"type":"v1","addr":"172.21.15.112:6803","nonce":2249725715}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":2249725715},{"type":"v1","addr":"172.21.15.112:6805","nonce":2249725715}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":2249725715},{"type":"v1","addr":"172.21.15.112:6809","nonce":2249725715}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":2249725715},{"type":"v1","addr":"172.21.15.112:6807","nonce":2249725715}]},"public_addr":"172.21.15.112:6803/2249725715","cluster_addr":"172.21.15.112:6805/2249725715","heartbeat_back_addr":"172.21.15.112:6809/2249725715","heartbeat_front_addr":"172.21.15.112:6807/2249725715","state":["exists","up"]},{"osd":1,"uuid":"baa30761-660d-452a-bdfe-00569457e9a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":20141175},{"type":"v1","addr":"172.21.15.112:6811","nonce":20141175}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":20141175},{"type":"v1","addr":"172.21.15.112:6813","nonce":20141175}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":20141175},{"type":"v1","addr":"172.21.15.112:6817","nonce":20141175}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":20141175},{"type":"v1","addr":"172.21.15.112:6815","nonce":20141175}]},"public_addr":"172.21.15.112:6811/20141175","cluster_addr":"172.21.15.112:6813/20141175","heartbeat_back_addr":"172.21.15.112:6817/20141175","heartbeat_front_addr":"172.21.15.112:6815/20141175","state":["exists","up"]},{"osd":2,"uuid":"17e98a3f-1edf-4186-8a34-2dce3e86d234","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":1768051982},{"type":"v1","addr":"172.21.15.112:6819","nonce":1768051982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":1768051982},{"type":"v1","addr":"172.21.15.112:6821","nonce":1768051982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":1768051982},{"type":"v1","addr":"172.21.15.112:6825","nonce":1768051982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":1768051982},{"type":"v1","addr":"172.21.15.112:6823","nonce":1768051982}]},"public_addr":"172.21.15.112:6819/1768051982","cluster_addr":"172.21.15.112:6821/1768051982","heartbeat_back_addr":"172.21.15.112:6825/1768051982","heartbeat_front_addr":"172.21.15.112:6823/1768051982","state":["exists","up"]},{"osd":3,"uuid":"6c85a093-d36e-4c61-bd75-95c2f09be643","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":1408970703},{"type":"v1","addr":"172.21.15.112:6827","nonce":1408970703}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":1408970703},{"type":"v1","addr":"172.21.15.112:6829","nonce":1408970703}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":1408970703},{"type":"v1","addr":"172.21.15.112:6833","nonce":1408970703}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":1408970703},{"type":"v1","addr":"172.21.15.112:6831","nonce":1408970703}]},"public_addr":"172.21.15.112:6827/1408970703","cluster_addr":"172.21.15.112:6829/1408970703","heartbeat_back_addr":"172.21.15.112:6833/1408970703","heartbeat_front_addr":"172.21.15.112:6831/1408970703","state":["exists","up"]},{"osd":4,"uuid":"31d857fe-771a-474b-8faf-bd38342cf00f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":3452220802},{"type":"v1","addr":"172.21.15.164:6801","nonce":3452220802}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":3452220802},{"type":"v1","addr":"172.21.15.164:6803","nonce":3452220802}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":3452220802},{"type":"v1","addr":"172.21.15.164:6807","nonce":3452220802}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":3452220802},{"type":"v1","addr":"172.21.15.164:6805","nonce":3452220802}]},"public_addr":"172.21.15.164:6801/3452220802","cluster_addr":"172.21.15.164:6803/3452220802","heartbeat_back_addr":"172.21.15.164:6807/3452220802","heartbeat_front_addr":"172.21.15.164:6805/3452220802","state":["exists","up"]},{"osd":5,"uuid":"5042dca7-9893-4bf1-9eac-9b01b34a2cd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":1097508373},{"type":"v1","addr":"172.21.15.164:6809","nonce":1097508373}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":1097508373},{"type":"v1","addr":"172.21.15.164:6811","nonce":1097508373}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":1097508373},{"type":"v1","addr":"172.21.15.164:6815","nonce":1097508373}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":1097508373},{"type":"v1","addr":"172.21.15.164:6813","nonce":1097508373}]},"public_addr":"172.21.15.164:6809/1097508373","cluster_addr":"172.21.15.164:6811/1097508373","heartbeat_back_addr":"172.21.15.164:6815/1097508373","heartbeat_front_addr":"172.21.15.164:6813/1097508373","state":["exists","up"]},{"osd":6,"uuid":"0e80c2ba-9b4b-4eef-a2ab-144c5ec8555c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":3245318540},{"type":"v1","addr":"172.21.15.164:6817","nonce":3245318540}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":3245318540},{"type":"v1","addr":"172.21.15.164:6819","nonce":3245318540}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":3245318540},{"type":"v1","addr":"172.21.15.164:6823","nonce":3245318540}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":3245318540},{"type":"v1","addr":"172.21.15.164:6821","nonce":3245318540}]},"public_addr":"172.21.15.164:6817/3245318540","cluster_addr":"172.21.15.164:6819/3245318540","heartbeat_back_addr":"172.21.15.164:6823/3245318540","heartbeat_front_addr":"172.21.15.164:6821/3245318540","state":["exists","up"]},{"osd":7,"uuid":"ec486d6f-2acf-4175-9ed2-5fbe142944c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":897796359},{"type":"v1","addr":"172.21.15.164:6825","nonce":897796359}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":897796359},{"type":"v1","addr":"172.21.15.164:6827","nonce":897796359}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":897796359},{"type":"v1","addr":"172.21.15.164:6831","nonce":897796359}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":897796359},{"type":"v1","addr":"172.21.15.164:6829","nonce":897796359}]},"public_addr":"172.21.15.164:6825/897796359","cluster_addr":"172.21.15.164:6827/897796359","heartbeat_back_addr":"172.21.15.164:6831/897796359","heartbeat_front_addr":"172.21.15.164:6829/897796359","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:23:33.156128+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:24:04.246089+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:24:34.348238+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:25:03.565446+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:25:26.088109+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:25:50.475116+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:26:15.272437+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:26:40.172061+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.112:6801/3398416842":"2024-04-27T15:22:27.152936+0000","172.21.15.112:0/447240260":"2024-04-27T15:22:27.152936+0000","172.21.15.112:0/1201901815":"2024-04-27T15:21:36.095747+0000","172.21.15.112:6800/3398416842":"2024-04-27T15:22:27.152936+0000","172.21.15.112:0/953234053":"2024-04-27T15:21:36.095747+0000","172.21.15.112:0/3385065922":"2024-04-27T15:22:27.152936+0000","172.21.15.112:0/4214048146":"2024-04-27T15:21:36.095747+0000","172.21.15.112:6801/1647678753":"2024-04-27T15:21:36.095747+0000","172.21.15.112:6800/1647678753":"2024-04-27T15:21:36.095747+0000","172.21.15.112:0/2379216906":"2024-04-27T15:21:06.648104+0000","172.21.15.112:0/3306085235":"2024-04-27T15:22:27.152936+0000","172.21.15.112:6801/478922692":"2024-04-27T15:21:06.648104+0000","172.21.15.112:0/3064986537":"2024-04-27T15:21:06.648104+0000","172.21.15.112:6800/478922692":"2024-04-27T15:21:06.648104+0000","172.21.15.112:0/2069494748":"2024-04-27T15:21:06.648104+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T15:26:58.958 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:58 smithi112 ceph-mon[21780]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-04-26T15:26:59.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:58 smithi164 ceph-mon[24045]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-04-26T15:26:59.604 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T15:26:59.604 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T15:26:59.605 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T15:26:59.605 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T15:26:59.605 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T15:26:59.606 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T15:26:59.606 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T15:26:59.606 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T15:26:59.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:26:59 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1438056720' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:27:00.004 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:00.007 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:00.007 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:00.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:26:59 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1438056720' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:27:00.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:00 smithi112 ceph-mon[21780]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-26T15:27:01.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:00 smithi164 ceph-mon[24045]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-26T15:27:02.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:02 smithi112 ceph-mon[21780]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:03.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:02 smithi164 ceph-mon[24045]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:03.253 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:03.253 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:03.255 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:03.256 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:03.257 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:04.996 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:04 smithi112 ceph-mon[21780]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:05.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:04 smithi164 ceph-mon[24045]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:06.413 INFO:teuthology.orchestra.run.smithi112.stdout:38654705708 2024-04-26T15:27:06.413 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T15:27:06.808 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:06 smithi112 ceph-mon[21780]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:07.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:06 smithi164 ceph-mon[24045]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:09.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:08 smithi164 ceph-mon[24045]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:09.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:08 smithi112 ceph-mon[21780]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:10.635 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:10.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:10 smithi112 ceph-mon[21780]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:11.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:10 smithi164 ceph-mon[24045]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:11.443 INFO:teuthology.orchestra.run.smithi112.stdout:137438953495 2024-04-26T15:27:11.443 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T15:27:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:12 smithi164 ceph-mon[24045]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:13.131 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:12 smithi112 ceph-mon[21780]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:14.003 INFO:teuthology.orchestra.run.smithi112.stdout:240518168585 2024-04-26T15:27:14.004 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T15:27:14.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:14 smithi112 ceph-mon[21780]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:14.991 INFO:teuthology.orchestra.run.smithi112.stdout:206158430222 2024-04-26T15:27:14.992 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T15:27:15.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:14 smithi164 ceph-mon[24045]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:15.965 INFO:teuthology.orchestra.run.smithi112.stdout:81604378659 2024-04-26T15:27:15.966 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T15:27:16.166 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:16.187 INFO:teuthology.orchestra.run.smithi112.stdout:60129542183 2024-04-26T15:27:16.187 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T15:27:16.748 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:16 smithi112 ceph-mon[21780]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:17.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:16 smithi164 ceph-mon[24045]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:18.212 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:18.238 INFO:teuthology.orchestra.run.smithi112.stdout:115964117020 2024-04-26T15:27:18.239 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T15:27:18.739 INFO:teuthology.orchestra.run.smithi112.stdout:171798691860 2024-04-26T15:27:18.740 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T15:27:19.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:18 smithi164 ceph-mon[24045]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:19.193 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:18 smithi112 ceph-mon[21780]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:19.603 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:19.603 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:19.603 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:20.921 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:20 smithi112 ceph-mon[21780]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:21.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:20 smithi164 ceph-mon[24045]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:21.894 INFO:teuthology.orchestra.run.smithi112.stdout:137438953497 2024-04-26T15:27:22.530 INFO:teuthology.orchestra.run.smithi112.stdout:38654705712 2024-04-26T15:27:22.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:22 smithi112 ceph-mon[21780]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:22.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:22 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/179145354' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T15:27:22.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:22 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/128567909' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T15:27:23.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:22 smithi164 ceph-mon[24045]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:23.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:22 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/179145354' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T15:27:23.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:22 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/128567909' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T15:27:24.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:24 smithi112 ceph-mon[21780]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:25.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:24 smithi164 ceph-mon[24045]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:25.110 INFO:teuthology.orchestra.run.smithi112.stdout:115964117022 2024-04-26T15:27:25.583 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953495 got 137438953497 for osd.4 2024-04-26T15:27:25.583 DEBUG:teuthology.parallel:result is None 2024-04-26T15:27:25.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:25 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1788951212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T15:27:26.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:25 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1788951212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T15:27:26.667 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117020 got 115964117022 for osd.3 2024-04-26T15:27:26.667 DEBUG:teuthology.parallel:result is None 2024-04-26T15:27:26.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:26 smithi112 ceph-mon[21780]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:27.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:26 smithi164 ceph-mon[24045]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:27.429 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:27.631 INFO:teuthology.orchestra.run.smithi112.stdout:240518168588 2024-04-26T15:27:27.812 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:27 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/3597129883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T15:27:27.812 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:27 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:27:27.812 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:27 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:27:27.963 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705708 got 38654705712 for osd.0 2024-04-26T15:27:27.963 DEBUG:teuthology.parallel:result is None 2024-04-26T15:27:28.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:27 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/3597129883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T15:27:28.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:27 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:27:28.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:27 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:27:28.258 INFO:teuthology.orchestra.run.smithi112.stdout:60129542188 2024-04-26T15:27:28.860 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542188 for osd.1 2024-04-26T15:27:28.860 DEBUG:teuthology.parallel:result is None 2024-04-26T15:27:29.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:28 smithi164 ceph-mon[24045]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:29.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:28 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1469729400' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T15:27:29.146 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:28 smithi112 ceph-mon[21780]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:29.146 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:28 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1469729400' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T15:27:29.988 INFO:teuthology.orchestra.run.smithi112.stdout:206158430226 2024-04-26T15:27:30.824 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:30 smithi112 ceph-mon[21780]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:30.824 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:30 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1056300057' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T15:27:31.002 INFO:teuthology.orchestra.run.smithi112.stdout:171798691862 2024-04-26T15:27:31.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:30 smithi164 ceph-mon[24045]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:31.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:30 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1056300057' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T15:27:31.233 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430222 got 206158430226 for osd.6 2024-04-26T15:27:31.233 DEBUG:teuthology.parallel:result is None 2024-04-26T15:27:31.795 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:31 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/3475683333' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T15:27:31.818 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691860 got 171798691862 for osd.5 2024-04-26T15:27:31.818 DEBUG:teuthology.parallel:result is None 2024-04-26T15:27:31.896 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:32.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:31 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/3475683333' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T15:27:32.342 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168585 got 240518168588 for osd.7 2024-04-26T15:27:32.342 DEBUG:teuthology.parallel:result is None 2024-04-26T15:27:32.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:32 smithi112 ceph-mon[21780]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:33.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:32 smithi164 ceph-mon[24045]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:34.126 INFO:teuthology.orchestra.run.smithi112.stdout:81604378663 2024-04-26T15:27:34.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:34 smithi112 ceph-mon[21780]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:34.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:34 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1581574343' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T15:27:34.992 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378659 got 81604378663 for osd.2 2024-04-26T15:27:34.992 DEBUG:teuthology.parallel:result is None 2024-04-26T15:27:34.992 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T15:27:34.993 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T15:27:35.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:34 smithi164 ceph-mon[24045]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:35.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:34 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1581574343' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T15:27:35.211 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:36.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:36 smithi112 ceph-mon[21780]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:37.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:36 smithi164 ceph-mon[24045]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:37.104 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:27:37.105 INFO:teuthology.orchestra.run.smithi112.stderr:dumped all 2024-04-26T15:27:37.757 INFO:teuthology.orchestra.run.smithi112.stdout:{"pg_ready":true,"pg_map":{"version":202,"stamp":"2024-04-26T15:27:35.731578+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":49136,"kb_used_data":6384,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912080,"statfs":{"total":767960285184,"available":767909969920,"internally_reserved":0,"allocated":6537216,"data_stored":3572274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001966"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-26T15:26:46.091413+0000","last_change":"2024-04-26T15:26:46.091413+0000","last_active":"2024-04-26T15:26:46.091413+0000","last_peered":"2024-04-26T15:26:46.091413+0000","last_clean":"2024-04-26T15:26:46.091413+0000","last_became_active":"2024-04-26T15:26:46.090990+0000","last_became_peered":"2024-04-26T15:26:46.090990+0000","last_unstale":"2024-04-26T15:26:46.091413+0000","last_undegraded":"2024-04-26T15:26:46.091413+0000","last_fullsized":"2024-04-26T15:26:46.091413+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T15:24:38.080218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T15:24:38.080218+0000","last_clean_scrub_stamp":"2024-04-26T15:24:38.080218+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-28T02:25:13.076238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2129920,"data_stored":2115712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36299999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46700000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51100000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54900000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59099999999999997}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6184,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738968,"statfs":{"total":95995035648,"available":95988703232,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.628}]},{"osd":1,"last update":"Fri Apr 26 15:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.52800000000000002}]},{"osd":2,"last update":"Fri Apr 26 15:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.68100000000000005}]},{"osd":3,"last update":"Fri Apr 26 15:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.70099999999999996}]},{"osd":4,"last update":"Fri Apr 26 15:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.74399999999999999}]},{"osd":5,"last update":"Fri Apr 26 15:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.71999999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78500000000000003}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:27:13 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.317,"15min":0.29299999999999998},"min":{"1min":0.20899999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.247},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.34599999999999997,"15min":0.312},"min":{"1min":0.23000000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.41499999999999998}]},{"osd":2,"last update":"Fri Apr 26 15:26:43 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.36199999999999999,"15min":0.35299999999999998},"min":{"1min":0.33200000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.374,"15min":0.36499999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.503,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.23400000000000001}]},{"osd":3,"last update":"Fri Apr 26 15:27:08 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.42499999999999999,"15min":0.42099999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":1.085,"5min":1.085,"15min":1.085},"last":0.379},{"interface":"front","average":{"1min":0.439,"5min":0.47199999999999998,"15min":0.47799999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.53400000000000003,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.33500000000000002}]},{"osd":4,"last update":"Fri Apr 26 15:27:30 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.64300000000000002,"15min":0.65400000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.874,"5min":1.8400000000000001,"15min":1.8400000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.55000000000000004,"15min":0.54800000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.40600000000000003}]},{"osd":5,"last update":"Fri Apr 26 15:26:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.64000000000000001}]},{"osd":6,"last update":"Fri Apr 26 15:27:21 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.42699999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67300000000000004}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 15:27:15 2024","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.317,"15min":0.29899999999999999},"min":{"1min":0.22,"5min":0.192,"15min":0.192},"max":{"1min":0.47699999999999998,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.376,"5min":0.33000000000000002,"15min":0.30499999999999999},"min":{"1min":0.247,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.55300000000000005,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.42099999999999999}]},{"osd":2,"last update":"Fri Apr 26 15:26:38 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.35499999999999998,"15min":0.33900000000000002},"min":{"1min":0.312,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.371,"15min":0.35799999999999998},"min":{"1min":0.32800000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.45500000000000002}]},{"osd":3,"last update":"Fri Apr 26 15:27:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.432,"15min":0.436},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.54900000000000004,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.313,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.60999999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.48199999999999998}]},{"osd":4,"last update":"Fri Apr 26 15:26:31 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.85099999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.745}]},{"osd":5,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.67400000000000004}]},{"osd":6,"last update":"Fri Apr 26 15:27:17 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.755},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.77700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68100000000000005}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:26:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.374,"15min":0.36599999999999999},"min":{"1min":0.28199999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.5},{"interface":"front","average":{"1min":0.437,"5min":0.38700000000000001,"15min":0.379},"min":{"1min":0.30099999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.57799999999999996,"5min":0.622,"15min":0.622},"last":0.45700000000000002}]},{"osd":1,"last update":"Fri Apr 26 15:26:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.36599999999999999,"15min":0.35199999999999998},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.433,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.30499999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.46899999999999997}]},{"osd":3,"last update":"Fri Apr 26 15:27:12 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.433,"15min":0.434},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.58899999999999997,"5min":0.625,"15min":0.625},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.441,"5min":0.44600000000000001,"15min":0.44700000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.52000000000000002}]},{"osd":4,"last update":"Fri Apr 26 15:27:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.60699999999999998,"15min":0.60199999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.871}]},{"osd":5,"last update":"Fri Apr 26 15:26:56 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.753}]},{"osd":6,"last update":"Fri Apr 26 15:27:22 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.80900000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.79600000000000004}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:27:05 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.32900000000000001}]},{"osd":1,"last update":"Fri Apr 26 15:27:05 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39000000000000001,"15min":0.38800000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.39000000000000001}]},{"osd":2,"last update":"Fri Apr 26 15:27:05 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.42099999999999999,"15min":0.41899999999999998},"min":{"1min":0.28999999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.432,"15min":0.435},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.379}]},{"osd":4,"last update":"Fri Apr 26 15:27:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.434},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.58299999999999996,"15min":0.58999999999999997},"min":{"1min":0.42699999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.81499999999999995,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.50800000000000001}]},{"osd":5,"last update":"Fri Apr 26 15:26:56 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.52300000000000002}]},{"osd":6,"last update":"Fri Apr 26 15:27:17 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.59099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5792,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739360,"statfs":{"total":95995035648,"available":95989104640,"internally_reserved":0,"allocated":614400,"data_stored":247944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.56499999999999995,"15min":0.56899999999999995},"min":{"1min":0.38,"5min":0.379,"15min":0.379},"max":{"1min":0.69799999999999995,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.56799999999999995,"15min":0.56699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.54700000000000004}]},{"osd":1,"last update":"Fri Apr 26 15:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.55200000000000005,"15min":0.55400000000000005},"min":{"1min":0.36199999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.54000000000000004,"15min":0.53700000000000003},"min":{"1min":0.36899999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.55600000000000005}]},{"osd":2,"last update":"Fri Apr 26 15:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42499999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.73799999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.54700000000000004,"15min":0.54000000000000004},"min":{"1min":0.46999999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.59299999999999997}]},{"osd":3,"last update":"Fri Apr 26 15:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.57299999999999995,"15min":0.57099999999999995},"min":{"1min":0.434,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.624},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.60999999999999999,"15min":0.61599999999999999},"min":{"1min":0.432,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.53700000000000003}]},{"osd":5,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.51300000000000001}]},{"osd":6,"last update":"Fri Apr 26 15:27:20 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.43099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47099999999999997}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.63100000000000001}]},{"osd":1,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.66000000000000003}]},{"osd":2,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.623}]},{"osd":3,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.70099999999999996}]},{"osd":4,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.64400000000000002}]},{"osd":6,"last update":"Fri Apr 26 15:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.54400000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58799999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T15:27:37.758 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T15:27:37.968 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:38.090 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:37 smithi112 ceph-mon[21780]: from='client.14446 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:27:38.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:37 smithi164 ceph-mon[24045]: from='client.14446 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:27:39.046 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:38 smithi112 ceph-mon[21780]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:39.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:38 smithi164 ceph-mon[24045]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:39.686 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:27:39.688 INFO:teuthology.orchestra.run.smithi112.stderr:dumped all 2024-04-26T15:27:40.311 INFO:teuthology.orchestra.run.smithi112.stdout:{"pg_ready":true,"pg_map":{"version":203,"stamp":"2024-04-26T15:27:37.731886+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":49136,"kb_used_data":6384,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912080,"statfs":{"total":767960285184,"available":767909969920,"internally_reserved":0,"allocated":6537216,"data_stored":3572274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001841"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-26T15:26:46.091413+0000","last_change":"2024-04-26T15:26:46.091413+0000","last_active":"2024-04-26T15:26:46.091413+0000","last_peered":"2024-04-26T15:26:46.091413+0000","last_clean":"2024-04-26T15:26:46.091413+0000","last_became_active":"2024-04-26T15:26:46.090990+0000","last_became_peered":"2024-04-26T15:26:46.090990+0000","last_unstale":"2024-04-26T15:26:46.091413+0000","last_undegraded":"2024-04-26T15:26:46.091413+0000","last_fullsized":"2024-04-26T15:26:46.091413+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T15:24:38.080218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T15:24:38.080218+0000","last_clean_scrub_stamp":"2024-04-26T15:24:38.080218+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-28T02:25:13.076238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2129920,"data_stored":2115712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36299999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46700000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51100000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54900000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59099999999999997}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6184,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738968,"statfs":{"total":95995035648,"available":95988703232,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.628}]},{"osd":1,"last update":"Fri Apr 26 15:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.52800000000000002}]},{"osd":2,"last update":"Fri Apr 26 15:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.68100000000000005}]},{"osd":3,"last update":"Fri Apr 26 15:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.70099999999999996}]},{"osd":4,"last update":"Fri Apr 26 15:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.74399999999999999}]},{"osd":5,"last update":"Fri Apr 26 15:27:18 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.71999999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78500000000000003}]}]},{"osd":1,"up_from":14,"seq":60129542190,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:27:13 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.317,"15min":0.29299999999999998},"min":{"1min":0.20899999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.34599999999999997,"15min":0.312},"min":{"1min":0.23000000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.33800000000000002}]},{"osd":2,"last update":"Fri Apr 26 15:26:43 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.36199999999999999,"15min":0.35299999999999998},"min":{"1min":0.33200000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.374,"15min":0.36499999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.503,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.32400000000000001}]},{"osd":3,"last update":"Fri Apr 26 15:27:08 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.42499999999999999,"15min":0.42099999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":1.085,"5min":1.085,"15min":1.085},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.439,"5min":0.47199999999999998,"15min":0.47799999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.53400000000000003,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.40600000000000003}]},{"osd":4,"last update":"Fri Apr 26 15:27:30 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.64300000000000002,"15min":0.65400000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.874,"5min":1.8400000000000001,"15min":1.8400000000000001},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.55000000000000004,"15min":0.54800000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.755}]},{"osd":5,"last update":"Fri Apr 26 15:26:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.77400000000000002}]},{"osd":6,"last update":"Fri Apr 26 15:27:21 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.73399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.79700000000000004}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 15:27:15 2024","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.317,"15min":0.29899999999999999},"min":{"1min":0.22,"5min":0.192,"15min":0.192},"max":{"1min":0.47699999999999998,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.31},{"interface":"front","average":{"1min":0.376,"5min":0.33000000000000002,"15min":0.30499999999999999},"min":{"1min":0.247,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.55300000000000005,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.23999999999999999}]},{"osd":2,"last update":"Fri Apr 26 15:26:38 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.35499999999999998,"15min":0.33900000000000002},"min":{"1min":0.312,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.371,"15min":0.35799999999999998},"min":{"1min":0.32800000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.29399999999999998}]},{"osd":3,"last update":"Fri Apr 26 15:27:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.432,"15min":0.436},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.54900000000000004,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.313,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.60999999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.32300000000000001}]},{"osd":4,"last update":"Fri Apr 26 15:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.55600000000000005,"15min":0.55200000000000005},"min":{"1min":0.41599999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.56899999999999995,"15min":0.56299999999999994},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.57199999999999995}]},{"osd":5,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.48399999999999999}]},{"osd":6,"last update":"Fri Apr 26 15:27:17 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.61799999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52100000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378664,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:26:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.374,"15min":0.36599999999999999},"min":{"1min":0.28199999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.435},{"interface":"front","average":{"1min":0.437,"5min":0.38700000000000001,"15min":0.379},"min":{"1min":0.30099999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.57799999999999996,"5min":0.622,"15min":0.622},"last":0.36099999999999999}]},{"osd":1,"last update":"Fri Apr 26 15:26:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.36599999999999999,"15min":0.35199999999999998},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.433,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.30499999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.38300000000000001}]},{"osd":3,"last update":"Fri Apr 26 15:27:12 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.433,"15min":0.434},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.58899999999999997,"5min":0.625,"15min":0.625},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.44600000000000001,"15min":0.44700000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.41099999999999998}]},{"osd":4,"last update":"Fri Apr 26 15:27:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.60699999999999998,"15min":0.60199999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.76000000000000001}]},{"osd":5,"last update":"Fri Apr 26 15:26:56 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.64200000000000002}]},{"osd":6,"last update":"Fri Apr 26 15:27:22 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.626}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65300000000000002}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:27:05 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.32900000000000001}]},{"osd":1,"last update":"Fri Apr 26 15:27:05 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39000000000000001,"15min":0.38800000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.39000000000000001}]},{"osd":2,"last update":"Fri Apr 26 15:27:05 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.42099999999999999,"15min":0.41899999999999998},"min":{"1min":0.28999999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.432,"15min":0.435},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.379}]},{"osd":4,"last update":"Fri Apr 26 15:27:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.434},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.58299999999999996,"15min":0.58999999999999997},"min":{"1min":0.42699999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.81499999999999995,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.50800000000000001}]},{"osd":5,"last update":"Fri Apr 26 15:26:56 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.52300000000000002}]},{"osd":6,"last update":"Fri Apr 26 15:27:17 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.59099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5792,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739360,"statfs":{"total":95995035648,"available":95989104640,"internally_reserved":0,"allocated":614400,"data_stored":247944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.56499999999999995,"15min":0.56899999999999995},"min":{"1min":0.38,"5min":0.379,"15min":0.379},"max":{"1min":0.69799999999999995,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.56799999999999995,"15min":0.56699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.54700000000000004}]},{"osd":1,"last update":"Fri Apr 26 15:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.55200000000000005,"15min":0.55400000000000005},"min":{"1min":0.36199999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.54000000000000004,"15min":0.53700000000000003},"min":{"1min":0.36899999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.55600000000000005}]},{"osd":2,"last update":"Fri Apr 26 15:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42499999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.73799999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.54700000000000004,"15min":0.54000000000000004},"min":{"1min":0.46999999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.59299999999999997}]},{"osd":3,"last update":"Fri Apr 26 15:27:31 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.57299999999999995,"15min":0.57099999999999995},"min":{"1min":0.434,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.624},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.60999999999999999,"15min":0.61599999999999999},"min":{"1min":0.432,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.53700000000000003}]},{"osd":5,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.51300000000000001}]},{"osd":6,"last update":"Fri Apr 26 15:27:20 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.43099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47099999999999997}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.63100000000000001}]},{"osd":1,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.66000000000000003}]},{"osd":2,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.623}]},{"osd":3,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.70099999999999996}]},{"osd":4,"last update":"Fri Apr 26 15:26:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.64400000000000002}]},{"osd":6,"last update":"Fri Apr 26 15:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.54400000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58799999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T15:27:40.312 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T15:27:40.312 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T15:27:40.312 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T15:27:40.312 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T15:27:40.523 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/mon.smithi112/config 2024-04-26T15:27:40.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:40 smithi112 ceph-mon[21780]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:27:40.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:40 smithi112 ceph-mon[21780]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:41.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:40 smithi164 ceph-mon[24045]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:27:41.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:40 smithi164 ceph-mon[24045]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:42.297 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:27:42.297 INFO:teuthology.orchestra.run.smithi112.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-26T15:27:42.932 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-26T15:27:42.933 INFO:tasks.cephadm:Setup complete, yielding 2024-04-26T15:27:42.933 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T15:27:42.947 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:27:42.947 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'ceph orch status' 2024-04-26T15:27:43.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:42 smithi164 ceph-mon[24045]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:43.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:42 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/324032430' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T15:27:43.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:42 smithi112 ceph-mon[21780]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:43.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:42 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/324032430' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T15:27:44.860 INFO:teuthology.orchestra.run.smithi112.stdout:Backend: cephadm 2024-04-26T15:27:44.860 INFO:teuthology.orchestra.run.smithi112.stdout:Available: Yes 2024-04-26T15:27:44.861 INFO:teuthology.orchestra.run.smithi112.stdout:Paused: No 2024-04-26T15:27:45.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:44 smithi164 ceph-mon[24045]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:27:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:27:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:27:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:27:45.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:44 smithi112 ceph-mon[21780]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:45.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:27:45.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:27:45.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:27:45.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:27:45.558 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'ceph orch ps' 2024-04-26T15:27:45.848 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:45 smithi112 ceph-mon[21780]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:27:46.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:45 smithi164 ceph-mon[24045]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:27:47.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:46 smithi164 ceph-mon[24045]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:47.116 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:46 smithi112 ceph-mon[21780]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:47.405 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:27:47.406 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (4m) 2m ago 5m 17.6M - ba2b418f427c c0d9c4d60bab 2024-04-26T15:27:47.406 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e c91aa1382673 2024-04-26T15:27:47.406 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (5m) 66s ago 5m 6400k - 17.2.0 e1d6a67b021e ed58e2be875d 2024-04-26T15:27:47.406 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (4m) 2m ago 5m 45.7M - 8.3.5 dad864ee21e9 16304abfa866 2024-04-26T15:27:47.406 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:9283 running (7m) 2m ago 6m 451M - 17.2.0 e1d6a67b021e 37fa85fc1610 2024-04-26T15:27:47.406 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283 running (5m) 66s ago 5m 413M - 17.2.0 e1d6a67b021e ab46b47c5040 2024-04-26T15:27:47.406 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (7m) 2m ago 7m 41.5M 2048M 17.2.0 e1d6a67b021e 44d7d076db9f 2024-04-26T15:27:47.407 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (4m) 66s ago 4m 40.5M 2048M 17.2.0 e1d6a67b021e 3e800cb663ea 2024-04-26T15:27:47.407 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (5m) 2m ago 5m 15.5M - 1dbe0e931976 7a4f240204de 2024-04-26T15:27:47.407 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (4m) 66s ago 4m 17.3M - 1dbe0e931976 0b5fa6213ccc 2024-04-26T15:27:47.407 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (4m) 2m ago 4m 33.7M 3216M 17.2.0 e1d6a67b021e 72c22bffb3c9 2024-04-26T15:27:47.407 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (3m) 2m ago 3m 34.6M 3216M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:27:47.407 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (3m) 2m ago 3m 34.6M 3216M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:27:47.407 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (2m) 2m ago 2m 27.9M 3216M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:27:47.407 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (2m) 66s ago 2m 36.4M 3984M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:27:47.408 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (2m) 66s ago 2m 35.1M 3984M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:27:47.408 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (96s) 66s ago 96s 31.7M 3984M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:27:47.408 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (71s) 66s ago 71s 13.4M 3984M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:27:47.408 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (4m) 2m ago 4m 54.2M - 514e6a882f6e ce3b8a2da81f 2024-04-26T15:27:48.043 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'ceph orch ls' 2024-04-26T15:27:48.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:48 smithi112 ceph-mon[21780]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:27:48.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:48 smithi112 ceph-mon[21780]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:49.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:48 smithi164 ceph-mon[24045]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:27:49.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:48 smithi164 ceph-mon[24045]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:49.970 INFO:teuthology.orchestra.run.smithi112.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-26T15:27:49.971 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-04-26T15:27:49.971 INFO:teuthology.orchestra.run.smithi112.stdout:crash 2/2 2m ago 6m * 2024-04-26T15:27:49.971 INFO:teuthology.orchestra.run.smithi112.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-04-26T15:27:49.971 INFO:teuthology.orchestra.run.smithi112.stdout:mgr 2/2 2m ago 6m count:2 2024-04-26T15:27:49.971 INFO:teuthology.orchestra.run.smithi112.stdout:mon 2/2 2m ago 5m smithi112:172.21.15.112=smithi112;smithi164:172.21.15.164=smithi164;count:2 2024-04-26T15:27:49.971 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-04-26T15:27:49.971 INFO:teuthology.orchestra.run.smithi112.stdout:osd 8 2m ago - 2024-04-26T15:27:49.971 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-04-26T15:27:50.759 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'ceph orch host ls' 2024-04-26T15:27:51.061 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:50 smithi112 ceph-mon[21780]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:51.061 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:50 smithi112 ceph-mon[21780]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:27:51.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:50 smithi164 ceph-mon[24045]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:50 smithi164 ceph-mon[24045]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:27:52.542 INFO:teuthology.orchestra.run.smithi112.stdout:HOST ADDR LABELS STATUS 2024-04-26T15:27:52.542 INFO:teuthology.orchestra.run.smithi112.stdout:smithi112 172.21.15.112 2024-04-26T15:27:52.542 INFO:teuthology.orchestra.run.smithi112.stdout:smithi164 172.21.15.164 2024-04-26T15:27:52.542 INFO:teuthology.orchestra.run.smithi112.stdout:2 hosts in cluster 2024-04-26T15:27:52.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:52 smithi112 ceph-mon[21780]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:53.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:52 smithi164 ceph-mon[24045]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:53.185 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'ceph orch device ls' 2024-04-26T15:27:54.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:53 smithi164 ceph-mon[24045]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:27:54.161 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:53 smithi112 ceph-mon[21780]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:27:54.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:54 smithi112 ceph-mon[21780]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:55.070 INFO:teuthology.orchestra.run.smithi112.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-04-26T15:27:55.070 INFO:teuthology.orchestra.run.smithi112.stdout:smithi112 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400NV400BGN 400G LVM detected, locked 2024-04-26T15:27:55.070 INFO:teuthology.orchestra.run.smithi112.stdout:smithi164 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT623300H0400BGN 400G LVM detected, locked 2024-04-26T15:27:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:54 smithi164 ceph-mon[24045]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:55.809 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:55 smithi112 ceph-mon[21780]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:27:55.835 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-26T15:27:55.872 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:27:55.873 DEBUG:teuthology.orchestra.run.smithi112:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-26T15:27:55.898 INFO:teuthology.orchestra.run.smithi112.stderr:+ systemctl stop nfs-server 2024-04-26T15:27:55.905 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi164.front.sepia.ceph.com 2024-04-26T15:27:55.906 DEBUG:teuthology.orchestra.run.smithi164:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-26T15:27:55.934 INFO:teuthology.orchestra.run.smithi164.stderr:+ systemctl stop nfs-server 2024-04-26T15:27:55.942 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T15:27:55.950 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:27:55.950 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-04-26T15:27:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:55 smithi164 ceph-mon[24045]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:27:56.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:56 smithi112 ceph-mon[21780]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:57.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:56 smithi164 ceph-mon[24045]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:59.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:58 smithi164 ceph-mon[24045]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:59.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:58 smithi164 ceph-mon[24045]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:27:59.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:58 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-26T15:27:59.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:58 smithi112 ceph-mon[21780]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:27:59.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:58 smithi112 ceph-mon[21780]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:27:59.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:58 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-26T15:28:00.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:59 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-26T15:28:00.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:59 smithi164 ceph-mon[24045]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T15:28:00.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:27:59 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-26T15:28:00.156 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:59 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-26T15:28:00.156 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:59 smithi112 ceph-mon[21780]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T15:28:00.157 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:59 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-26T15:28:00.157 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:27:59 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112[21776]: 2024-04-26T15:27:59.808+0000 7f98ef2a8700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T15:28:00.617 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-04-26T15:28:00.627 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-04-26T15:28:00.627 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-26T15:28:00.949 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: pgmap v215: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:28:00.949 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-26T15:28:00.949 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T15:28:00.949 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-26T15:28:00.949 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T15:28:00.949 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T15:28:00.949 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T15:28:00.949 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-26T15:28:00.949 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: fsmap foofs:0 2024-04-26T15:28:00.949 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: Saving service mds.foofs spec with placement count:2 2024-04-26T15:28:00.950 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:00.950 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:00.950 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:00.950 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:00.950 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:00.950 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:00.950 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.fqmway", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T15:28:00.950 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.fqmway", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T15:28:00.950 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:00.950 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:00 smithi112 ceph-mon[21780]: Deploying daemon mds.foofs.smithi112.fqmway on smithi112 2024-04-26T15:28:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: pgmap v215: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:28:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-26T15:28:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T15:28:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-26T15:28:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T15:28:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T15:28:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T15:28:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-26T15:28:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: fsmap foofs:0 2024-04-26T15:28:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: Saving service mds.foofs spec with placement count:2 2024-04-26T15:28:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:01.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:01.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:01.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:01.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:01.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:01.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.fqmway", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T15:28:01.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.fqmway", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T15:28:01.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:01.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:00 smithi164 ceph-mon[24045]: Deploying daemon mds.foofs.smithi112.fqmway on smithi112 2024-04-26T15:28:01.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:01 smithi112 ceph-mon[21780]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T15:28:02.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:01 smithi164 ceph-mon[24045]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T15:28:02.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:02 smithi112 ceph-mon[21780]: pgmap v219: 65 pgs: 6 creating+peering, 12 active+clean, 47 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:28:02.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:02 smithi112 ceph-mon[21780]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T15:28:03.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:02 smithi164 ceph-mon[24045]: pgmap v219: 65 pgs: 6 creating+peering, 12 active+clean, 47 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:28:03.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:02 smithi164 ceph-mon[24045]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T15:28:03.098 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:28:03.098 INFO:teuthology.orchestra.run.smithi112.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-26T15:21:25.497200Z", "last_refresh": "2024-04-26T15:25:13.536323Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-26T15:21:18.687347Z", "last_refresh": "2024-04-26T15:25:13.536385Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-26T15:21:22.362098Z", "last_refresh": "2024-04-26T15:25:13.536445Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-26T15:27:59.832278Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-26T15:27:59.824257Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-26T15:21:17.390738Z", "last_refresh": "2024-04-26T15:25:13.536240Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi112:172.21.15.112=smithi112", "smithi164:172.21.15.164=smithi164"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-26T15:22:10.216652Z", "last_refresh": "2024-04-26T15:25:13.536102Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-26T15:21:23.960767Z", "last_refresh": "2024-04-26T15:25:13.536503Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-26T15:25:13.536620Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-26T15:21:20.469679Z", "last_refresh": "2024-04-26T15:25:13.536561Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-26T15:28:04.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:03 smithi164 ceph-mon[24045]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:28:04.179 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:03 smithi112 ceph-mon[21780]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:28:04.214 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-26T15:28:05.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:04 smithi164 ceph-mon[24045]: pgmap v221: 65 pgs: 6 creating+peering, 59 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:28:05.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:04 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:05.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:04 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.gbxewm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T15:28:05.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:04 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.gbxewm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T15:28:05.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:04 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:05.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:04 smithi112 ceph-mon[21780]: pgmap v221: 65 pgs: 6 creating+peering, 59 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:28:05.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:04 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:05.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:04 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.gbxewm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T15:28:05.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:04 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.gbxewm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T15:28:05.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:04 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:05.215 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-26T15:28:06.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:05 smithi164 ceph-mon[24045]: Deploying daemon mds.foofs.smithi164.gbxewm on smithi164 2024-04-26T15:28:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:05 smithi164 ceph-mon[24045]: mds.? [v2:172.21.15.112:6834/365553452,v1:172.21.15.112:6835/365553452] up:boot 2024-04-26T15:28:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:05 smithi164 ceph-mon[24045]: daemon mds.foofs.smithi112.fqmway assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-26T15:28:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:05 smithi164 ceph-mon[24045]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T15:28:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:05 smithi164 ceph-mon[24045]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T15:28:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:05 smithi164 ceph-mon[24045]: Cluster is now healthy 2024-04-26T15:28:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:05 smithi164 ceph-mon[24045]: fsmap foofs:0 1 up:standby 2024-04-26T15:28:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:05 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:28:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:05 smithi164 ceph-mon[24045]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:creating} 2024-04-26T15:28:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:05 smithi164 ceph-mon[24045]: daemon mds.foofs.smithi112.fqmway is now active in filesystem foofs as rank 0 2024-04-26T15:28:06.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:05 smithi112 ceph-mon[21780]: Deploying daemon mds.foofs.smithi164.gbxewm on smithi164 2024-04-26T15:28:06.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:05 smithi112 ceph-mon[21780]: mds.? [v2:172.21.15.112:6834/365553452,v1:172.21.15.112:6835/365553452] up:boot 2024-04-26T15:28:06.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:05 smithi112 ceph-mon[21780]: daemon mds.foofs.smithi112.fqmway assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-26T15:28:06.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:05 smithi112 ceph-mon[21780]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T15:28:06.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:05 smithi112 ceph-mon[21780]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T15:28:06.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:05 smithi112 ceph-mon[21780]: Cluster is now healthy 2024-04-26T15:28:06.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:05 smithi112 ceph-mon[21780]: fsmap foofs:0 1 up:standby 2024-04-26T15:28:06.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:05 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:28:06.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:05 smithi112 ceph-mon[21780]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:creating} 2024-04-26T15:28:06.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:05 smithi112 ceph-mon[21780]: daemon mds.foofs.smithi112.fqmway is now active in filesystem foofs as rank 0 2024-04-26T15:28:06.940 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:06 smithi112 ceph-mon[21780]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:28:06.940 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:06 smithi112 ceph-mon[21780]: mds.? [v2:172.21.15.112:6834/365553452,v1:172.21.15.112:6835/365553452] up:active 2024-04-26T15:28:06.940 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:06 smithi112 ceph-mon[21780]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 2024-04-26T15:28:06.954 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:06 smithi164 ceph-mon[24045]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:28:06.954 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:06 smithi164 ceph-mon[24045]: mds.? [v2:172.21.15.112:6834/365553452,v1:172.21.15.112:6835/365553452] up:active 2024-04-26T15:28:06.954 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:06 smithi164 ceph-mon[24045]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 2024-04-26T15:28:07.498 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:28:07.499 INFO:teuthology.orchestra.run.smithi112.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-26T15:21:25.497200Z", "last_refresh": "2024-04-26T15:25:13.536323Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-26T15:21:18.687347Z", "last_refresh": "2024-04-26T15:25:13.536385Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-26T15:21:22.362098Z", "last_refresh": "2024-04-26T15:25:13.536445Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-26T15:27:59.832278Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-26T15:27:59.824257Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-26T15:21:17.390738Z", "last_refresh": "2024-04-26T15:25:13.536240Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi112:172.21.15.112=smithi112", "smithi164:172.21.15.164=smithi164"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-26T15:22:10.216652Z", "last_refresh": "2024-04-26T15:25:13.536102Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-26T15:21:23.960767Z", "last_refresh": "2024-04-26T15:25:13.536503Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-26T15:25:13.536620Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-26T15:21:20.469679Z", "last_refresh": "2024-04-26T15:25:13.536561Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-26T15:28:08.136 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-26T15:28:08.312 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:08 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:08.312 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:08 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:08.312 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:08 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:08.312 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:08 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:08.312 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:08 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:08.312 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:08 smithi164 ceph-mon[24045]: mds.? [v2:172.21.15.164:6832/3623172766,v1:172.21.15.164:6833/3623172766] up:boot 2024-04-26T15:28:08.312 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:08 smithi164 ceph-mon[24045]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 1 up:standby 2024-04-26T15:28:08.312 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:08 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.gbxewm"}]: dispatch 2024-04-26T15:28:08.313 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:08 smithi164 ceph-mon[24045]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 1 up:standby 2024-04-26T15:28:08.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:08 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:08.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:08 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:08.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:08 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:08.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:08 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:08.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:08 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:08.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:08 smithi112 ceph-mon[21780]: mds.? [v2:172.21.15.164:6832/3623172766,v1:172.21.15.164:6833/3623172766] up:boot 2024-04-26T15:28:08.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:08 smithi112 ceph-mon[21780]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 1 up:standby 2024-04-26T15:28:08.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:08 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.gbxewm"}]: dispatch 2024-04-26T15:28:08.407 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:08 smithi112 ceph-mon[21780]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 1 up:standby 2024-04-26T15:28:09.137 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-26T15:28:09.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:09 smithi112 ceph-mon[21780]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:28:09.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:09 smithi112 ceph-mon[21780]: pgmap v223: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T15:28:09.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:09 smithi164 ceph-mon[24045]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:28:09.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:09 smithi164 ceph-mon[24045]: pgmap v223: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T15:28:10.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:10 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:10.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:10 smithi112 ceph-mon[21780]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-04-26T15:28:10.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:10 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:10.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:10 smithi164 ceph-mon[24045]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-04-26T15:28:12.441 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:28:12.441 INFO:teuthology.orchestra.run.smithi112.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-26T15:21:25.497200Z", "last_refresh": "2024-04-26T15:25:13.536323Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-26T15:21:18.687347Z", "last_refresh": "2024-04-26T15:25:13.536385Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-26T15:21:22.362098Z", "last_refresh": "2024-04-26T15:25:13.536445Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-26T15:27:59.832278Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-26T15:27:59.824257Z", "last_refresh": "2024-04-26T15:28:09.325171Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-26T15:21:17.390738Z", "last_refresh": "2024-04-26T15:25:13.536240Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi112:172.21.15.112=smithi112", "smithi164:172.21.15.164=smithi164"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-26T15:22:10.216652Z", "last_refresh": "2024-04-26T15:25:13.536102Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-26T15:21:23.960767Z", "last_refresh": "2024-04-26T15:25:13.536503Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-26T15:25:13.536620Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-26T15:21:20.469679Z", "last_refresh": "2024-04-26T15:25:13.536561Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-26T15:28:13.089 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:12 smithi112 ceph-mon[21780]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-26T15:28:13.090 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:12 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:12 smithi164 ceph-mon[24045]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-26T15:28:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:12 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:13.112 INFO:tasks.cephadm:mds.foofs has 1/2 2024-04-26T15:28:13.952 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:13 smithi112 ceph-mon[21780]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:28:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:13 smithi164 ceph-mon[24045]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:28:14.114 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-26T15:28:14.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:14 smithi112 ceph-mon[21780]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T15:28:14.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:14.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:14.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:14.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:14.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:14.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:14 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:15.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:14 smithi164 ceph-mon[24045]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T15:28:15.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:14 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:16.117 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:28:16.117 INFO:teuthology.orchestra.run.smithi112.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-26T15:21:25.497200Z", "last_refresh": "2024-04-26T15:28:14.361869Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-26T15:21:18.687347Z", "last_refresh": "2024-04-26T15:28:09.324567Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-26T15:21:22.362098Z", "last_refresh": "2024-04-26T15:28:14.362006Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-26T15:27:59.832278Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-26T15:27:59.824257Z", "last_refresh": "2024-04-26T15:28:09.325171Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-26T15:21:17.390738Z", "last_refresh": "2024-04-26T15:28:09.324701Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi112:172.21.15.112=smithi112", "smithi164:172.21.15.164=smithi164"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-26T15:22:10.216652Z", "last_refresh": "2024-04-26T15:28:09.324769Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-26T15:21:23.960767Z", "last_refresh": "2024-04-26T15:28:09.324831Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-26T15:28:09.324892Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-26T15:21:20.469679Z", "last_refresh": "2024-04-26T15:28:14.362132Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-26T15:28:16.790 INFO:tasks.cephadm:mds.foofs has 2/2 2024-04-26T15:28:16.790 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T15:28:16.800 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:28:16.801 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-04-26T15:28:17.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:16 smithi164 ceph-mon[24045]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T15:28:17.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:16 smithi164 ceph-mon[24045]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:28:17.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:16 smithi112 ceph-mon[21780]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T15:28:17.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:16 smithi112 ceph-mon[21780]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:28:19.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:18 smithi164 ceph-mon[24045]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T15:28:19.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:18 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-26T15:28:19.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:18 smithi112 ceph-mon[21780]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T15:28:19.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:18 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-26T15:28:20.065 INFO:teuthology.orchestra.run.smithi112.stdout:NFS Cluster Created Successfully 2024-04-26T15:28:20.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:20 smithi164 ceph-mon[24045]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:20.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-26T15:28:20.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:20 smithi164 ceph-mon[24045]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T15:28:20.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-26T15:28:20.369 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:20 smithi112 ceph-mon[21780]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:20.369 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-26T15:28:20.369 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:20 smithi112 ceph-mon[21780]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T15:28:20.369 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-26T15:28:20.845 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-04-26T15:28:21.107 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: pgmap v230: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T15:28:21.107 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-26T15:28:21.107 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T15:28:21.107 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: Saving service nfs.foo spec with placement count:2 2024-04-26T15:28:21.107 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:21.107 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:21.107 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:21.107 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:21.107 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:21.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:21.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: Creating key for client.nfs.foo.0.0.smithi112.qswmgy 2024-04-26T15:28:21.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi112.qswmgy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-26T15:28:21.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi112.qswmgy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-26T15:28:21.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-26T15:28:21.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-26T15:28:21.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-26T15:28:21.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:21.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-26T15:28:21.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-26T15:28:21.109 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi112.qswmgy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:28:21.109 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi112.qswmgy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-26T15:28:21.109 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:21 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:21.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: pgmap v230: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T15:28:21.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-26T15:28:21.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T15:28:21.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: Saving service nfs.foo spec with placement count:2 2024-04-26T15:28:21.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:21.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:21.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:21.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:21.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:21.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:21.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: Creating key for client.nfs.foo.0.0.smithi112.qswmgy 2024-04-26T15:28:21.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi112.qswmgy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-26T15:28:21.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi112.qswmgy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-26T15:28:21.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-26T15:28:21.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-26T15:28:21.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-26T15:28:21.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:21.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-26T15:28:21.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-26T15:28:21.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi112.qswmgy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:28:21.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi112.qswmgy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-26T15:28:21.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:21 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:22.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:22 smithi164 ceph-mon[24045]: Rados config object exists: conf-nfs.foo 2024-04-26T15:28:22.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:22 smithi164 ceph-mon[24045]: Creating key for client.nfs.foo.0.0.smithi112.qswmgy-rgw 2024-04-26T15:28:22.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:22 smithi164 ceph-mon[24045]: Deploying daemon nfs.foo.0.0.smithi112.qswmgy on smithi112 2024-04-26T15:28:22.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:22 smithi164 ceph-mon[24045]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T15:28:22.381 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:22 smithi112 ceph-mon[21780]: Rados config object exists: conf-nfs.foo 2024-04-26T15:28:22.381 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:22 smithi112 ceph-mon[21780]: Creating key for client.nfs.foo.0.0.smithi112.qswmgy-rgw 2024-04-26T15:28:22.381 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:22 smithi112 ceph-mon[21780]: Deploying daemon nfs.foo.0.0.smithi112.qswmgy on smithi112 2024-04-26T15:28:22.381 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:22 smithi112 ceph-mon[21780]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T15:28:23.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:23 smithi164 ceph-mon[24045]: pgmap v233: 97 pgs: 20 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-26T15:28:23.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:23 smithi164 ceph-mon[24045]: mgrmap e20: smithi112.llioug(active, since 5m), standbys: smithi164.tuhwus 2024-04-26T15:28:23.362 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:23 smithi112 ceph-mon[21780]: pgmap v233: 97 pgs: 20 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-26T15:28:23.362 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:23 smithi112 ceph-mon[21780]: mgrmap e20: smithi112.llioug(active, since 5m), standbys: smithi164.tuhwus 2024-04-26T15:28:24.117 INFO:teuthology.orchestra.run.smithi112.stderr:Invalid command: Unexpected argument '--clusterid' 2024-04-26T15:28:24.117 INFO:teuthology.orchestra.run.smithi112.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-04-26T15:28:24.117 INFO:teuthology.orchestra.run.smithi112.stderr:Error EINVAL: invalid command 2024-04-26T15:28:24.453 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:28:24.453 INFO:teuthology.orchestra.run.smithi112.stdout: "bind": "/fake", 2024-04-26T15:28:24.453 INFO:teuthology.orchestra.run.smithi112.stdout: "fs": "foofs", 2024-04-26T15:28:24.453 INFO:teuthology.orchestra.run.smithi112.stdout: "path": "/", 2024-04-26T15:28:24.453 INFO:teuthology.orchestra.run.smithi112.stdout: "cluster": "foo", 2024-04-26T15:28:24.453 INFO:teuthology.orchestra.run.smithi112.stdout: "mode": "RW" 2024-04-26T15:28:24.454 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:28:25.174 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:25 smithi112 ceph-mon[21780]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-26T15:28:25.174 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:25 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-26T15:28:25.174 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:25 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-26T15:28:25.174 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:25 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:25.174 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:25 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.lkwjnc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-26T15:28:25.174 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:25 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.lkwjnc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-26T15:28:25.174 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:25 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-26T15:28:25.174 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:25 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-26T15:28:25.174 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:25 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:25.197 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-04-26T15:28:25.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:25 smithi164 ceph-mon[24045]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-26T15:28:25.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:25 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-26T15:28:25.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:25 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-26T15:28:25.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:25 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:25.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:25 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.lkwjnc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-26T15:28:25.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:25 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.lkwjnc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-26T15:28:25.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:25 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-26T15:28:25.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:25 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-26T15:28:25.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:25 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:26.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:26 smithi164 ceph-mon[24045]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:26.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:26 smithi164 ceph-mon[24045]: Creating key for client.nfs.foo.1.0.smithi164.lkwjnc 2024-04-26T15:28:26.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:26 smithi164 ceph-mon[24045]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-26T15:28:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:26 smithi112 ceph-mon[21780]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:26 smithi112 ceph-mon[21780]: Creating key for client.nfs.foo.1.0.smithi164.lkwjnc 2024-04-26T15:28:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:26 smithi112 ceph-mon[21780]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-26T15:28:27.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:27 smithi164 ceph-mon[24045]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-26T15:28:27.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:27 smithi112 ceph-mon[21780]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-26T15:28:28.205 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:28 smithi164 ceph-mon[24045]: from='client.14542 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:28.205 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-26T15:28:28.206 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-26T15:28:28.206 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.lkwjnc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:28:28.206 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.lkwjnc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-26T15:28:28.206 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:28.206 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-04-26T15:28:28.206 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-04-26T15:28:28.206 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-04-26T15:28:28.206 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T15:28:28.206 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:28:28.206 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:28 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:28:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:28 smithi112 ceph-mon[21780]: from='client.14542 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-26T15:28:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-26T15:28:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.lkwjnc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:28:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.lkwjnc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-26T15:28:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-04-26T15:28:28.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-04-26T15:28:28.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-04-26T15:28:28.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T15:28:28.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:28:28.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:28 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:28:29.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:29 smithi164 ceph-mon[24045]: Rados config object exists: conf-nfs.foo 2024-04-26T15:28:29.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:29 smithi164 ceph-mon[24045]: Creating key for client.nfs.foo.1.0.smithi164.lkwjnc-rgw 2024-04-26T15:28:29.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:29 smithi164 ceph-mon[24045]: Deploying daemon nfs.foo.1.0.smithi164.lkwjnc on smithi164 2024-04-26T15:28:29.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:29 smithi164 ceph-mon[24045]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-26T15:28:29.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:29 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-04-26T15:28:29.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:29 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-04-26T15:28:29.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:29 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-04-26T15:28:29.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:29 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-26T15:28:29.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:29 smithi164 ceph-mon[24045]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T15:28:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:29 smithi112 ceph-mon[21780]: Rados config object exists: conf-nfs.foo 2024-04-26T15:28:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:29 smithi112 ceph-mon[21780]: Creating key for client.nfs.foo.1.0.smithi164.lkwjnc-rgw 2024-04-26T15:28:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:29 smithi112 ceph-mon[21780]: Deploying daemon nfs.foo.1.0.smithi164.lkwjnc on smithi164 2024-04-26T15:28:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:29 smithi112 ceph-mon[21780]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-26T15:28:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:29 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-04-26T15:28:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:29 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-04-26T15:28:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:29 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-04-26T15:28:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:29 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-26T15:28:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:29 smithi112 ceph-mon[21780]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T15:28:30.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:30 smithi164 ceph-mon[24045]: from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:30.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:30 smithi164 ceph-mon[24045]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T15:28:30.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:30 smithi112 ceph-mon[21780]: from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:30.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:30 smithi112 ceph-mon[21780]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T15:28:31.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:31 smithi164 ceph-mon[24045]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:31.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:31 smithi164 ceph-mon[24045]: pgmap v239: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-04-26T15:28:31.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:31.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:31.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:31.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:31 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:31 smithi112 ceph-mon[21780]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:31 smithi112 ceph-mon[21780]: pgmap v239: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-04-26T15:28:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:31.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:31.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:31 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:32.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:32 smithi112 ceph-mon[21780]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:32.473 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:32 smithi164 ceph-mon[24045]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:33.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:33 smithi112 ceph-mon[21780]: pgmap v240: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.6 KiB/s wr, 8 op/s 2024-04-26T15:28:33.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:33 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:33.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:33 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:33.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:33 smithi164 ceph-mon[24045]: pgmap v240: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.6 KiB/s wr, 8 op/s 2024-04-26T15:28:33.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:33 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:33.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:33 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:34.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:34 smithi112 ceph-mon[21780]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:34.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:34 smithi164 ceph-mon[24045]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:34.985 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-04-26T15:28:35.352 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:35 smithi112 ceph-mon[21780]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:35.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:35 smithi112 ceph-mon[21780]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-04-26T15:28:35.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:35.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:35.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:35.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:35.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:35.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:35.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:35.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:35 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:35 smithi164 ceph-mon[24045]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:35 smithi164 ceph-mon[24045]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-04-26T15:28:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:35 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:35.750 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-26T15:28:35.760 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:28:35.760 DEBUG:teuthology.orchestra.run.smithi112:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-04-26T15:28:35.784 INFO:teuthology.orchestra.run.smithi112.stderr:+ mkdir /mnt/foo 2024-04-26T15:28:35.793 DEBUG:teuthology.orchestra.run.smithi112:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-04-26T15:28:35.856 INFO:teuthology.orchestra.run.smithi112.stderr:++ hostname 2024-04-26T15:28:35.856 INFO:teuthology.orchestra.run.smithi112.stderr:+ mount -t nfs smithi112:/fake /mnt/foo -o sync 2024-04-26T15:28:36.242 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:36 smithi112 ceph-mon[21780]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:36.244 DEBUG:teuthology.orchestra.run.smithi112:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-04-26T15:28:36.307 INFO:teuthology.orchestra.run.smithi112.stderr:+ echo test 2024-04-26T15:28:36.327 DEBUG:teuthology.orchestra.run.smithi112:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-04-26T15:28:36.389 INFO:teuthology.orchestra.run.smithi112.stderr:+ sync 2024-04-26T15:28:36.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:36 smithi164 ceph-mon[24045]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:28:36.625 INFO:teuthology.run_tasks:Running task parallel... 2024-04-26T15:28:36.635 INFO:teuthology.task.parallel:starting parallel... 2024-04-26T15:28:36.635 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-26T15:28:36.636 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-26T15:28:36.636 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:28:36.636 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-26T15:28:36.637 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-26T15:28:36.637 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-04-26T15:28:36.637 INFO:teuthology.task.exec:Executing custom commands... 2024-04-26T15:28:36.638 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:28:36.638 DEBUG:teuthology.orchestra.run.smithi112:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-04-26T15:28:36.696 INFO:teuthology.orchestra.run.smithi112.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-26T15:28:36.696 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:28:36.696 INFO:teuthology.orchestra.run.smithi112.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-04-26T15:28:36.697 INFO:teuthology.orchestra.run.smithi112.stdout:4 of 5 processes prepared for launch 0 sec 2024-04-26T15:28:36.797 INFO:teuthology.orchestra.run.smithi112.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-26T15:28:36.797 INFO:teuthology.orchestra.run.smithi112.stdout:releasing clients 2024-04-26T15:28:37.458 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:37 smithi112 ceph-mon[21780]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-04-26T15:28:37.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:37 smithi164 ceph-mon[24045]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-04-26T15:28:37.797 INFO:teuthology.orchestra.run.smithi112.stdout: 5 153 32.26 MB/sec warmup 1 sec latency 55.504 ms 2024-04-26T15:28:38.797 INFO:teuthology.orchestra.run.smithi112.stdout: 5 317 28.88 MB/sec warmup 2 sec latency 24.231 ms 2024-04-26T15:28:39.258 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:39 smithi112 ceph-mon[21780]: pgmap v243: 97 pgs: 97 active+clean; 5.1 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 509 KiB/s wr, 13 op/s 2024-04-26T15:28:39.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:39 smithi164 ceph-mon[24045]: pgmap v243: 97 pgs: 97 active+clean; 5.1 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 509 KiB/s wr, 13 op/s 2024-04-26T15:28:39.797 INFO:teuthology.orchestra.run.smithi112.stdout: 5 467 27.21 MB/sec warmup 3 sec latency 32.336 ms 2024-04-26T15:28:40.196 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-26T15:28:40.797 INFO:teuthology.orchestra.run.smithi112.stdout: 5 670 26.16 MB/sec warmup 4 sec latency 24.607 ms 2024-04-26T15:28:41.310 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:41 smithi112 ceph-mon[21780]: pgmap v244: 97 pgs: 97 active+clean; 20 MiB data, 60 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.9 MiB/s wr, 118 op/s 2024-04-26T15:28:41.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:41 smithi164 ceph-mon[24045]: pgmap v244: 97 pgs: 97 active+clean; 20 MiB data, 60 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.9 MiB/s wr, 118 op/s 2024-04-26T15:28:41.797 INFO:teuthology.orchestra.run.smithi112.stdout: 5 1098 22.75 MB/sec warmup 5 sec latency 31.297 ms 2024-04-26T15:28:42.797 INFO:teuthology.orchestra.run.smithi112.stdout: 5 2054 22.15 MB/sec warmup 6 sec latency 23.300 ms 2024-04-26T15:28:43.322 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:43 smithi112 ceph-mon[21780]: pgmap v245: 97 pgs: 97 active+clean; 61 MiB data, 96 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 5.2 MiB/s wr, 230 op/s 2024-04-26T15:28:43.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:43 smithi164 ceph-mon[24045]: pgmap v245: 97 pgs: 97 active+clean; 61 MiB data, 96 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 5.2 MiB/s wr, 230 op/s 2024-04-26T15:28:43.797 INFO:teuthology.orchestra.run.smithi112.stdout: 5 3036 23.88 MB/sec warmup 7 sec latency 21.250 ms 2024-04-26T15:28:44.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 3760 24.60 MB/sec warmup 8 sec latency 24.869 ms 2024-04-26T15:28:45.233 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-26T15:28:45.358 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:45 smithi112 ceph-mon[21780]: pgmap v246: 97 pgs: 97 active+clean; 111 MiB data, 194 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 9.9 MiB/s wr, 450 op/s 2024-04-26T15:28:45.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:45 smithi164 ceph-mon[24045]: pgmap v246: 97 pgs: 97 active+clean; 111 MiB data, 194 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 9.9 MiB/s wr, 450 op/s 2024-04-26T15:28:45.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 4201 22.35 MB/sec warmup 9 sec latency 63.417 ms 2024-04-26T15:28:46.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 4910 21.98 MB/sec warmup 10 sec latency 19.535 ms 2024-04-26T15:28:47.381 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:47 smithi112 ceph-mon[21780]: pgmap v247: 97 pgs: 97 active+clean; 115 MiB data, 189 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 11 MiB/s wr, 527 op/s 2024-04-26T15:28:47.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:47 smithi164 ceph-mon[24045]: pgmap v247: 97 pgs: 97 active+clean; 115 MiB data, 189 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 11 MiB/s wr, 527 op/s 2024-04-26T15:28:47.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 5783 21.07 MB/sec warmup 11 sec latency 36.909 ms 2024-04-26T15:28:48.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 6468 21.70 MB/sec warmup 12 sec latency 32.115 ms 2024-04-26T15:28:49.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:49 smithi112 ceph-mon[21780]: pgmap v248: 97 pgs: 97 active+clean; 123 MiB data, 207 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 12 MiB/s wr, 671 op/s 2024-04-26T15:28:49.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:49 smithi164 ceph-mon[24045]: pgmap v248: 97 pgs: 97 active+clean; 123 MiB data, 207 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 12 MiB/s wr, 671 op/s 2024-04-26T15:28:49.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 7137 21.97 MB/sec warmup 13 sec latency 31.266 ms 2024-04-26T15:28:50.071 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph mgr module enable nfs --force' 2024-04-26T15:28:50.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 7490 21.11 MB/sec warmup 14 sec latency 36.173 ms 2024-04-26T15:28:51.416 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:51 smithi112 ceph-mon[21780]: pgmap v249: 97 pgs: 97 active+clean; 129 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 14 MiB/s wr, 854 op/s 2024-04-26T15:28:51.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:51 smithi164 ceph-mon[24045]: pgmap v249: 97 pgs: 97 active+clean; 129 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 14 MiB/s wr, 854 op/s 2024-04-26T15:28:51.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 7925 19.90 MB/sec warmup 15 sec latency 54.074 ms 2024-04-26T15:28:52.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 8243 19.10 MB/sec warmup 16 sec latency 42.682 ms 2024-04-26T15:28:53.241 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:53 smithi112 ceph-mon[21780]: pgmap v250: 97 pgs: 97 active+clean; 134 MiB data, 285 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 14 MiB/s wr, 911 op/s 2024-04-26T15:28:53.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:53 smithi164 ceph-mon[24045]: pgmap v250: 97 pgs: 97 active+clean; 134 MiB data, 285 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 14 MiB/s wr, 911 op/s 2024-04-26T15:28:53.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 8917 19.00 MB/sec warmup 17 sec latency 30.301 ms 2024-04-26T15:28:54.204 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:54 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/160231598' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-26T15:28:54.205 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:54 smithi112 ceph-mon[21780]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-26T15:28:54.227 INFO:teuthology.orchestra.run.smithi112.stderr:module 'nfs' is already enabled 2024-04-26T15:28:54.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:54 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/160231598' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-26T15:28:54.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:54 smithi164 ceph-mon[24045]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-26T15:28:54.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 9685 18.80 MB/sec warmup 18 sec latency 20.184 ms 2024-04-26T15:28:55.455 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:55 smithi112 ceph-mon[21780]: pgmap v251: 97 pgs: 97 active+clean; 145 MiB data, 356 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 13 MiB/s wr, 991 op/s 2024-04-26T15:28:55.456 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:55 smithi112 ceph-mon[21780]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-26T15:28:55.456 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:55 smithi112 ceph-mon[21780]: mgrmap e21: smithi112.llioug(active, since 6m), standbys: smithi164.tuhwus 2024-04-26T15:28:55.458 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-26T15:28:55.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:55 smithi164 ceph-mon[24045]: pgmap v251: 97 pgs: 97 active+clean; 145 MiB data, 356 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 13 MiB/s wr, 991 op/s 2024-04-26T15:28:55.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:55 smithi164 ceph-mon[24045]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-26T15:28:55.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:55 smithi164 ceph-mon[24045]: mgrmap e21: smithi112.llioug(active, since 6m), standbys: smithi164.tuhwus 2024-04-26T15:28:55.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 10208 18.99 MB/sec warmup 19 sec latency 49.422 ms 2024-04-26T15:28:56.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 10817 19.33 MB/sec warmup 20 sec latency 45.224 ms 2024-04-26T15:28:57.241 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:57 smithi112 ceph-mon[21780]: pgmap v252: 97 pgs: 97 active+clean; 143 MiB data, 360 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.5 MiB/s wr, 790 op/s 2024-04-26T15:28:57.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:57 smithi164 ceph-mon[24045]: pgmap v252: 97 pgs: 97 active+clean; 143 MiB data, 360 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.5 MiB/s wr, 790 op/s 2024-04-26T15:28:57.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 11241 18.68 MB/sec warmup 21 sec latency 40.835 ms 2024-04-26T15:28:58.798 INFO:teuthology.orchestra.run.smithi112.stdout: 5 11750 18.29 MB/sec warmup 22 sec latency 46.535 ms 2024-04-26T15:28:59.001 INFO:teuthology.orchestra.run.smithi112.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T15:28:59.266 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:59 smithi112 ceph-mon[21780]: pgmap v253: 97 pgs: 97 active+clean; 143 MiB data, 399 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 778 op/s 2024-04-26T15:28:59.266 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:59 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:59.266 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:59 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:59.266 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:59 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:59.266 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:28:59 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:59.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:59 smithi164 ceph-mon[24045]: pgmap v253: 97 pgs: 97 active+clean; 143 MiB data, 399 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 778 op/s 2024-04-26T15:28:59.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:59 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:28:59.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:59 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:28:59.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:59 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:28:59.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:28:59 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:28:59.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 12411 18.18 MB/sec warmup 23 sec latency 26.562 ms 2024-04-26T15:29:00.275 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:00 smithi112 ceph-mon[21780]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:00.275 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:00 smithi112 ceph-mon[21780]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T15:29:00.275 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:29:00.275 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:00 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:29:00.300 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-26T15:29:00.300 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:29:00.301 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-26T15:29:00.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:00 smithi164 ceph-mon[24045]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:00.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:00 smithi164 ceph-mon[24045]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T15:29:00.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:29:00.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:00 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:29:00.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 13025 17.73 MB/sec warmup 24 sec latency 37.972 ms 2024-04-26T15:29:01.295 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:01 smithi112 ceph-mon[21780]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T15:29:01.296 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:01 smithi112 ceph-mon[21780]: pgmap v254: 97 pgs: 97 active+clean; 151 MiB data, 411 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.0 MiB/s wr, 791 op/s 2024-04-26T15:29:01.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:01 smithi164 ceph-mon[24045]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T15:29:01.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:01 smithi164 ceph-mon[24045]: pgmap v254: 97 pgs: 97 active+clean; 151 MiB data, 411 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.0 MiB/s wr, 791 op/s 2024-04-26T15:29:01.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 13445 17.76 MB/sec warmup 25 sec latency 29.612 ms 2024-04-26T15:29:02.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 14088 18.12 MB/sec warmup 26 sec latency 23.920 ms 2024-04-26T15:29:03.336 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:03 smithi112 ceph-mon[21780]: pgmap v255: 97 pgs: 97 active+clean; 151 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.8 MiB/s wr, 718 op/s 2024-04-26T15:29:03.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:03 smithi164 ceph-mon[24045]: pgmap v255: 97 pgs: 97 active+clean; 151 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.8 MiB/s wr, 718 op/s 2024-04-26T15:29:03.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 14497 17.99 MB/sec warmup 27 sec latency 39.995 ms 2024-04-26T15:29:04.299 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:29:04.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 14950 17.49 MB/sec warmup 28 sec latency 30.638 ms 2024-04-26T15:29:05.344 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:05 smithi112 ceph-mon[21780]: pgmap v256: 97 pgs: 97 active+clean; 158 MiB data, 493 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 8.9 MiB/s wr, 831 op/s 2024-04-26T15:29:05.344 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:05 smithi112 ceph-mon[21780]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:05.359 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:29:05.359 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (5m) 30s ago 7m 25.2M - ba2b418f427c c0d9c4d60bab 2024-04-26T15:29:05.359 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (7m) 30s ago 7m 6405k - 17.2.0 e1d6a67b021e c91aa1382673 2024-04-26T15:29:05.359 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (6m) 32s ago 6m 6400k - 17.2.0 e1d6a67b021e ed58e2be875d 2024-04-26T15:29:05.359 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (5m) 30s ago 6m 52.4M - 8.3.5 dad864ee21e9 16304abfa866 2024-04-26T15:29:05.359 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (61s) 30s ago 60s 13.2M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (58s) 32s ago 58s 12.0M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:9283 running (8m) 30s ago 8m 463M - 17.2.0 e1d6a67b021e 37fa85fc1610 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283 running (6m) 32s ago 6m 414M - 17.2.0 e1d6a67b021e ab46b47c5040 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (8m) 30s ago 8m 43.3M 2048M 17.2.0 e1d6a67b021e 44d7d076db9f 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (6m) 32s ago 6m 38.1M 2048M 17.2.0 e1d6a67b021e 3e800cb663ea 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (41s) 30s ago 40s 51.9M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (35s) 32s ago 35s 50.7M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (6m) 30s ago 6m 15.3M - 1dbe0e931976 7a4f240204de 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (6m) 32s ago 6m 16.8M - 1dbe0e931976 0b5fa6213ccc 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (5m) 30s ago 5m 42.6M 3216M 17.2.0 e1d6a67b021e 72c22bffb3c9 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (5m) 30s ago 5m 42.8M 3216M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (4m) 30s ago 4m 43.3M 3216M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (4m) 30s ago 4m 42.8M 3216M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (3m) 32s ago 3m 43.4M 3984M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:29:05.360 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (3m) 32s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:29:05.361 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (2m) 32s ago 2m 40.1M 3984M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:29:05.361 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (2m) 32s ago 2m 44.0M 3984M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:29:05.361 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (5m) 30s ago 5m 63.1M - 514e6a882f6e ce3b8a2da81f 2024-04-26T15:29:05.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:05 smithi164 ceph-mon[24045]: pgmap v256: 97 pgs: 97 active+clean; 158 MiB data, 493 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 8.9 MiB/s wr, 831 op/s 2024-04-26T15:29:05.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:05 smithi164 ceph-mon[24045]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:05.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 15724 17.55 MB/sec warmup 29 sec latency 11.975 ms 2024-04-26T15:29:06.014 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:29:06.014 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:29:06.014 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:29:06.014 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:29:06.014 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:29:06.014 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:29:06.014 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:29:06.014 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:29:06.014 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T15:29:06.014 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:29:06.014 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:29:06.014 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:29:06.015 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:29:06.015 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:29:06.015 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-26T15:29:06.015 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:29:06.015 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:29:06.347 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:06 smithi112 ceph-mon[21780]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:06.348 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:06 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/519576285' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:29:06.567 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:29:06.568 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", 2024-04-26T15:29:06.568 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:29:06.568 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [], 2024-04-26T15:29:06.568 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "", 2024-04-26T15:29:06.568 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 image" 2024-04-26T15:29:06.568 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:29:06.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:06 smithi164 ceph-mon[24045]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:06.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:06 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/519576285' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:29:06.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 16425 17.33 MB/sec warmup 30 sec latency 27.716 ms 2024-04-26T15:29:07.208 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:29:07.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:07 smithi112 ceph-mon[21780]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:07.352 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:07 smithi112 ceph-mon[21780]: pgmap v257: 97 pgs: 97 active+clean; 160 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.9 MiB/s wr, 639 op/s 2024-04-26T15:29:07.352 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:07 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/399697746' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:29:07.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:07 smithi164 ceph-mon[24045]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:07.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:07 smithi164 ceph-mon[24045]: pgmap v257: 97 pgs: 97 active+clean; 160 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.9 MiB/s wr, 639 op/s 2024-04-26T15:29:07.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:07 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/399697746' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:29:07.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 17136 17.61 MB/sec warmup 31 sec latency 14.005 ms 2024-04-26T15:29:08.358 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:08 smithi112 ceph-mon[21780]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:08.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:08 smithi164 ceph-mon[24045]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:08.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 17814 17.92 MB/sec warmup 32 sec latency 22.358 ms 2024-04-26T15:29:09.363 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:09 smithi112 ceph-mon[21780]: pgmap v258: 97 pgs: 97 active+clean; 165 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 8.4 MiB/s wr, 832 op/s 2024-04-26T15:29:09.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:09 smithi164 ceph-mon[24045]: pgmap v258: 97 pgs: 97 active+clean; 165 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 8.4 MiB/s wr, 832 op/s 2024-04-26T15:29:09.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 18142 17.65 MB/sec warmup 33 sec latency 44.819 ms 2024-04-26T15:29:10.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 18632 17.26 MB/sec warmup 34 sec latency 36.635 ms 2024-04-26T15:29:11.374 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:11 smithi112 ceph-mon[21780]: pgmap v259: 97 pgs: 97 active+clean; 169 MiB data, 613 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.7 MiB/s wr, 913 op/s 2024-04-26T15:29:11.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:11 smithi164 ceph-mon[24045]: pgmap v259: 97 pgs: 97 active+clean; 169 MiB data, 613 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.7 MiB/s wr, 913 op/s 2024-04-26T15:29:11.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 19459 17.41 MB/sec warmup 35 sec latency 22.084 ms 2024-04-26T15:29:12.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 20399 17.40 MB/sec warmup 36 sec latency 21.938 ms 2024-04-26T15:29:13.283 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:13 smithi112 ceph-mon[21780]: pgmap v260: 97 pgs: 97 active+clean; 172 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 797 op/s 2024-04-26T15:29:13.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:13 smithi164 ceph-mon[24045]: pgmap v260: 97 pgs: 97 active+clean; 172 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 797 op/s 2024-04-26T15:29:13.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 21382 18.08 MB/sec warmup 37 sec latency 16.272 ms 2024-04-26T15:29:14.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 21904 17.88 MB/sec warmup 38 sec latency 22.100 ms 2024-04-26T15:29:15.489 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:15 smithi112 ceph-mon[21780]: pgmap v261: 97 pgs: 97 active+clean; 182 MiB data, 669 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 9.8 MiB/s wr, 949 op/s 2024-04-26T15:29:15.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:15 smithi164 ceph-mon[24045]: pgmap v261: 97 pgs: 97 active+clean; 182 MiB data, 669 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 9.8 MiB/s wr, 949 op/s 2024-04-26T15:29:15.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 22635 17.75 MB/sec warmup 39 sec latency 17.243 ms 2024-04-26T15:29:16.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 23551 17.74 MB/sec warmup 40 sec latency 21.099 ms 2024-04-26T15:29:17.501 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:17 smithi112 ceph-mon[21780]: pgmap v262: 97 pgs: 97 active+clean; 182 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.8 MiB/s wr, 714 op/s 2024-04-26T15:29:17.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:17 smithi164 ceph-mon[24045]: pgmap v262: 97 pgs: 97 active+clean; 182 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.8 MiB/s wr, 714 op/s 2024-04-26T15:29:17.800 INFO:teuthology.orchestra.run.smithi112.stdout: 5 24371 18.16 MB/sec warmup 41 sec latency 22.568 ms 2024-04-26T15:29:18.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 25182 18.38 MB/sec warmup 42 sec latency 18.686 ms 2024-04-26T15:29:19.510 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:19 smithi112 ceph-mon[21780]: pgmap v263: 97 pgs: 97 active+clean; 186 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 9.7 MiB/s wr, 920 op/s 2024-04-26T15:29:19.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:19 smithi164 ceph-mon[24045]: pgmap v263: 97 pgs: 97 active+clean; 186 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 9.7 MiB/s wr, 920 op/s 2024-04-26T15:29:19.800 INFO:teuthology.orchestra.run.smithi112.stdout: 5 25849 18.18 MB/sec warmup 43 sec latency 23.826 ms 2024-04-26T15:29:20.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 26478 18.13 MB/sec warmup 44 sec latency 15.920 ms 2024-04-26T15:29:21.519 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:21 smithi112 ceph-mon[21780]: pgmap v264: 97 pgs: 97 active+clean; 196 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 938 op/s 2024-04-26T15:29:21.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:21 smithi164 ceph-mon[24045]: pgmap v264: 97 pgs: 97 active+clean; 196 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 938 op/s 2024-04-26T15:29:21.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 27210 18.01 MB/sec warmup 45 sec latency 24.157 ms 2024-04-26T15:29:22.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 27976 18.24 MB/sec warmup 46 sec latency 21.521 ms 2024-04-26T15:29:23.280 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:23 smithi112 ceph-mon[21780]: pgmap v265: 97 pgs: 97 active+clean; 194 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 851 op/s 2024-04-26T15:29:23.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:23 smithi164 ceph-mon[24045]: pgmap v265: 97 pgs: 97 active+clean; 194 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 851 op/s 2024-04-26T15:29:23.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 28775 18.47 MB/sec warmup 47 sec latency 21.053 ms 2024-04-26T15:29:24.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 29404 18.28 MB/sec warmup 48 sec latency 23.185 ms 2024-04-26T15:29:25.304 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:25 smithi112 ceph-mon[21780]: pgmap v266: 97 pgs: 97 active+clean; 207 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 12 MiB/s wr, 1.15k op/s 2024-04-26T15:29:25.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:25 smithi164 ceph-mon[24045]: pgmap v266: 97 pgs: 97 active+clean; 207 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 12 MiB/s wr, 1.15k op/s 2024-04-26T15:29:25.799 INFO:teuthology.orchestra.run.smithi112.stdout: 5 29951 18.23 MB/sec warmup 49 sec latency 25.846 ms 2024-04-26T15:29:26.800 INFO:teuthology.orchestra.run.smithi112.stdout: 5 30561 18.06 MB/sec warmup 50 sec latency 32.754 ms 2024-04-26T15:29:26.815 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:26 smithi112 ceph-mon[21780]: pgmap v267: 97 pgs: 97 active+clean; 206 MiB data, 860 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.8 MiB/s wr, 907 op/s 2024-04-26T15:29:27.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:26 smithi164 ceph-mon[24045]: pgmap v267: 97 pgs: 97 active+clean; 206 MiB data, 860 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.8 MiB/s wr, 907 op/s 2024-04-26T15:29:27.800 INFO:teuthology.orchestra.run.smithi112.stdout: 5 31070 17.93 MB/sec warmup 51 sec latency 43.133 ms 2024-04-26T15:29:27.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:27 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:29:27.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:27 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:29:28.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:27 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:29:28.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:27 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:29:28.800 INFO:teuthology.orchestra.run.smithi112.stdout: 5 31766 18.15 MB/sec warmup 52 sec latency 34.571 ms 2024-04-26T15:29:28.843 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:28 smithi112 ceph-mon[21780]: pgmap v268: 97 pgs: 97 active+clean; 206 MiB data, 887 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 947 op/s 2024-04-26T15:29:29.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:28 smithi164 ceph-mon[24045]: pgmap v268: 97 pgs: 97 active+clean; 206 MiB data, 887 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 947 op/s 2024-04-26T15:29:29.800 INFO:teuthology.orchestra.run.smithi112.stdout: 5 32294 18.18 MB/sec warmup 53 sec latency 23.213 ms 2024-04-26T15:29:30.800 INFO:teuthology.orchestra.run.smithi112.stdout: 5 32721 17.93 MB/sec warmup 54 sec latency 22.569 ms 2024-04-26T15:29:30.823 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:30 smithi112 ceph-mon[21780]: pgmap v269: 97 pgs: 97 active+clean; 210 MiB data, 926 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 967 op/s 2024-04-26T15:29:31.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:30 smithi164 ceph-mon[24045]: pgmap v269: 97 pgs: 97 active+clean; 210 MiB data, 926 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 967 op/s 2024-04-26T15:29:31.800 INFO:teuthology.orchestra.run.smithi112.stdout: 5 33314 17.79 MB/sec warmup 55 sec latency 21.216 ms 2024-04-26T15:29:32.800 INFO:teuthology.orchestra.run.smithi112.stdout: 5 34043 17.83 MB/sec warmup 56 sec latency 18.703 ms 2024-04-26T15:29:32.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:32 smithi112 ceph-mon[21780]: pgmap v270: 97 pgs: 97 active+clean; 219 MiB data, 952 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 857 op/s 2024-04-26T15:29:33.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:32 smithi164 ceph-mon[24045]: pgmap v270: 97 pgs: 97 active+clean; 219 MiB data, 952 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 857 op/s 2024-04-26T15:29:33.802 INFO:teuthology.orchestra.run.smithi112.stdout: 5 34817 17.91 MB/sec warmup 57 sec latency 63.451 ms 2024-04-26T15:29:34.800 INFO:teuthology.orchestra.run.smithi112.stdout: 5 35594 18.13 MB/sec warmup 58 sec latency 26.761 ms 2024-04-26T15:29:34.853 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:34 smithi112 ceph-mon[21780]: pgmap v271: 97 pgs: 97 active+clean; 224 MiB data, 994 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 978 op/s 2024-04-26T15:29:35.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:34 smithi164 ceph-mon[24045]: pgmap v271: 97 pgs: 97 active+clean; 224 MiB data, 994 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 978 op/s 2024-04-26T15:29:35.800 INFO:teuthology.orchestra.run.smithi112.stdout: 5 36023 18.01 MB/sec warmup 59 sec latency 25.441 ms 2024-04-26T15:29:36.800 INFO:teuthology.orchestra.run.smithi112.stdout: 5 36409 17.80 MB/sec warmup 60 sec latency 101.718 ms 2024-04-26T15:29:36.866 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:36 smithi112 ceph-mon[21780]: pgmap v272: 97 pgs: 97 active+clean; 222 MiB data, 1000 MiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.4 MiB/s wr, 658 op/s 2024-04-26T15:29:37.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:36 smithi164 ceph-mon[24045]: pgmap v272: 97 pgs: 97 active+clean; 222 MiB data, 1000 MiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.4 MiB/s wr, 658 op/s 2024-04-26T15:29:37.745 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:29:37.805 INFO:teuthology.orchestra.run.smithi112.stdout: 5 37078 17.79 MB/sec warmup 61 sec latency 70.468 ms 2024-04-26T15:29:38.791 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (6m) 63s ago 7m 25.2M - ba2b418f427c c0d9c4d60bab 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (7m) 63s ago 7m 6405k - 17.2.0 e1d6a67b021e c91aa1382673 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (6m) 65s ago 6m 6400k - 17.2.0 e1d6a67b021e ed58e2be875d 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (5m) 63s ago 7m 52.4M - 8.3.5 dad864ee21e9 16304abfa866 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (94s) 63s ago 94s 13.2M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (92s) 65s ago 91s 12.0M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:9283 running (8m) 63s ago 8m 463M - 17.2.0 e1d6a67b021e 37fa85fc1610 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283 running (6m) 65s ago 6m 414M - 17.2.0 e1d6a67b021e ab46b47c5040 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (8m) 63s ago 8m 43.3M 2048M 17.2.0 e1d6a67b021e 44d7d076db9f 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (6m) 65s ago 6m 38.1M 2048M 17.2.0 e1d6a67b021e 3e800cb663ea 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (74s) 63s ago 74s 51.9M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (69s) 65s ago 68s 50.7M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (7m) 63s ago 7m 15.3M - 1dbe0e931976 7a4f240204de 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (6m) 65s ago 6m 16.8M - 1dbe0e931976 0b5fa6213ccc 2024-04-26T15:29:38.792 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (6m) 63s ago 6m 42.6M 3216M 17.2.0 e1d6a67b021e 72c22bffb3c9 2024-04-26T15:29:38.793 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (5m) 63s ago 5m 42.8M 3216M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:29:38.793 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (5m) 63s ago 5m 43.3M 3216M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:29:38.793 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (4m) 63s ago 4m 42.8M 3216M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:29:38.793 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (4m) 65s ago 4m 43.4M 3984M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:29:38.793 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (3m) 65s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:29:38.793 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (3m) 65s ago 3m 40.1M 3984M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:29:38.793 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (3m) 65s ago 3m 44.0M 3984M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:29:38.793 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (6m) 63s ago 6m 63.1M - 514e6a882f6e ce3b8a2da81f 2024-04-26T15:29:38.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 37875 17.71 MB/sec warmup 62 sec latency 21.292 ms 2024-04-26T15:29:38.876 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:38 smithi112 ceph-mon[21780]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:38.877 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:38 smithi112 ceph-mon[21780]: pgmap v273: 97 pgs: 97 active+clean; 226 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.6 MiB/s wr, 768 op/s 2024-04-26T15:29:38.877 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:38 smithi112 ceph-mon[21780]: from='client.24401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:39.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:38 smithi164 ceph-mon[24045]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:39.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:38 smithi164 ceph-mon[24045]: pgmap v273: 97 pgs: 97 active+clean; 226 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.6 MiB/s wr, 768 op/s 2024-04-26T15:29:39.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:38 smithi164 ceph-mon[24045]: from='client.24401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:39.430 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:29:39.430 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:29:39.430 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:29:39.430 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:29:39.430 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:29:39.430 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:29:39.430 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:29:39.430 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:29:39.430 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T15:29:39.430 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:29:39.431 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:29:39.431 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:29:39.431 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:29:39.431 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:29:39.431 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-26T15:29:39.431 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:29:39.431 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:29:39.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 38513 17.83 MB/sec warmup 63 sec latency 21.989 ms 2024-04-26T15:29:39.880 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:39 smithi112 ceph-mon[21780]: from='client.24405 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:39.881 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:39 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2202082959' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:29:39.937 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:29:39.938 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", 2024-04-26T15:29:39.938 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:29:39.938 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [], 2024-04-26T15:29:39.938 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "", 2024-04-26T15:29:39.938 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 image" 2024-04-26T15:29:39.938 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:29:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:39 smithi164 ceph-mon[24045]: from='client.24405 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:39 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/2202082959' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:29:40.469 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:29:40.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 39171 17.93 MB/sec warmup 64 sec latency 85.862 ms 2024-04-26T15:29:40.890 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:40 smithi112 ceph-mon[21780]: pgmap v274: 97 pgs: 97 active+clean; 235 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.3 MiB/s wr, 909 op/s 2024-04-26T15:29:40.890 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:40 smithi112 ceph-mon[21780]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:40.891 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:40 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/4192086855' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:29:41.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:40 smithi164 ceph-mon[24045]: pgmap v274: 97 pgs: 97 active+clean; 235 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.3 MiB/s wr, 909 op/s 2024-04-26T15:29:41.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:40 smithi164 ceph-mon[24045]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:29:41.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:40 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/4192086855' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:29:41.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 39681 17.82 MB/sec warmup 65 sec latency 18.854 ms 2024-04-26T15:29:42.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 40281 17.68 MB/sec warmup 66 sec latency 19.812 ms 2024-04-26T15:29:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:43 smithi112 ceph-mon[21780]: pgmap v275: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 8.4 MiB/s wr, 716 op/s 2024-04-26T15:29:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:43 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:29:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:43 smithi112 ceph-mon[21780]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (unknown) 2024-04-26T15:29:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:43 smithi112 ceph-mon[21780]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T15:29:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:43 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:29:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:43 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:29:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:43 smithi112 ceph-mon[21780]: Upgrade: Need to upgrade myself (mgr.smithi112.llioug) 2024-04-26T15:29:43.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:43 smithi164 ceph-mon[24045]: pgmap v275: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 8.4 MiB/s wr, 716 op/s 2024-04-26T15:29:43.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:43 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:29:43.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:43 smithi164 ceph-mon[24045]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (unknown) 2024-04-26T15:29:43.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:43 smithi164 ceph-mon[24045]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T15:29:43.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:43 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:29:43.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:43 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:29:43.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:43 smithi164 ceph-mon[24045]: Upgrade: Need to upgrade myself (mgr.smithi112.llioug) 2024-04-26T15:29:43.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 40971 17.74 MB/sec warmup 67 sec latency 30.324 ms 2024-04-26T15:29:44.155 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:44 smithi112 ceph-mon[21780]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 on smithi164 2024-04-26T15:29:44.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:44 smithi164 ceph-mon[24045]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 on smithi164 2024-04-26T15:29:44.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 41726 17.69 MB/sec warmup 68 sec latency 27.740 ms 2024-04-26T15:29:45.165 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:45 smithi112 ceph-mon[21780]: pgmap v276: 97 pgs: 97 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 891 op/s 2024-04-26T15:29:45.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:45 smithi164 ceph-mon[24045]: pgmap v276: 97 pgs: 97 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 891 op/s 2024-04-26T15:29:45.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 42479 17.88 MB/sec warmup 69 sec latency 29.621 ms 2024-04-26T15:29:46.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 43051 17.89 MB/sec warmup 70 sec latency 28.489 ms 2024-04-26T15:29:46.940 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:46 smithi112 ceph-mon[21780]: pgmap v277: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.3 MiB/s wr, 738 op/s 2024-04-26T15:29:47.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:46 smithi164 ceph-mon[24045]: pgmap v277: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.3 MiB/s wr, 738 op/s 2024-04-26T15:29:47.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 43477 17.72 MB/sec warmup 71 sec latency 40.937 ms 2024-04-26T15:29:48.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 43902 17.57 MB/sec warmup 72 sec latency 30.507 ms 2024-04-26T15:29:48.842 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:48 smithi112 ceph-mon[21780]: pgmap v278: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.9 MiB/s wr, 921 op/s 2024-04-26T15:29:49.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:48 smithi164 ceph-mon[24045]: pgmap v278: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.9 MiB/s wr, 921 op/s 2024-04-26T15:29:49.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 44451 17.56 MB/sec warmup 73 sec latency 76.319 ms 2024-04-26T15:29:50.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 45078 17.47 MB/sec warmup 74 sec latency 37.857 ms 2024-04-26T15:29:50.950 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:50 smithi112 ceph-mon[21780]: pgmap v279: 97 pgs: 97 active+clean; 262 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 957 op/s 2024-04-26T15:29:51.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:50 smithi164 ceph-mon[24045]: pgmap v279: 97 pgs: 97 active+clean; 262 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 957 op/s 2024-04-26T15:29:51.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 45586 17.51 MB/sec warmup 75 sec latency 59.768 ms 2024-04-26T15:29:52.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 46305 17.63 MB/sec warmup 76 sec latency 34.390 ms 2024-04-26T15:29:52.976 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:52 smithi112 ceph-mon[21780]: pgmap v280: 97 pgs: 97 active+clean; 262 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.9 MiB/s wr, 866 op/s 2024-04-26T15:29:53.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:52 smithi164 ceph-mon[24045]: pgmap v280: 97 pgs: 97 active+clean; 262 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.9 MiB/s wr, 866 op/s 2024-04-26T15:29:53.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 46629 17.52 MB/sec warmup 77 sec latency 49.823 ms 2024-04-26T15:29:54.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 47171 17.39 MB/sec warmup 78 sec latency 25.009 ms 2024-04-26T15:29:54.999 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:54 smithi112 ceph-mon[21780]: pgmap v281: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.7 MiB/s wr, 947 op/s 2024-04-26T15:29:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:54 smithi164 ceph-mon[24045]: pgmap v281: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.7 MiB/s wr, 947 op/s 2024-04-26T15:29:55.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 47720 17.33 MB/sec warmup 79 sec latency 54.537 ms 2024-04-26T15:29:56.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 48412 17.30 MB/sec warmup 80 sec latency 22.573 ms 2024-04-26T15:29:57.013 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:56 smithi112 ceph-mon[21780]: pgmap v282: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 666 op/s 2024-04-26T15:29:57.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:56 smithi164 ceph-mon[24045]: pgmap v282: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 666 op/s 2024-04-26T15:29:57.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 48945 17.28 MB/sec warmup 81 sec latency 44.006 ms 2024-04-26T15:29:58.808 INFO:teuthology.orchestra.run.smithi112.stdout: 5 49576 17.39 MB/sec warmup 82 sec latency 43.309 ms 2024-04-26T15:29:59.035 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:29:58 smithi112 ceph-mon[21780]: pgmap v283: 97 pgs: 97 active+clean; 275 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.9 MiB/s wr, 929 op/s 2024-04-26T15:29:59.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:29:58 smithi164 ceph-mon[24045]: pgmap v283: 97 pgs: 97 active+clean; 275 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.9 MiB/s wr, 929 op/s 2024-04-26T15:29:59.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 50042 17.37 MB/sec warmup 83 sec latency 80.795 ms 2024-04-26T15:30:00.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 50732 17.29 MB/sec warmup 84 sec latency 19.654 ms 2024-04-26T15:30:01.057 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:00 smithi112 ceph-mon[21780]: pgmap v284: 97 pgs: 97 active+clean; 272 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 768 op/s 2024-04-26T15:30:01.057 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:00 smithi112 ceph-mon[21780]: overall HEALTH_OK 2024-04-26T15:30:01.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:00 smithi164 ceph-mon[24045]: pgmap v284: 97 pgs: 97 active+clean; 272 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 768 op/s 2024-04-26T15:30:01.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:00 smithi164 ceph-mon[24045]: overall HEALTH_OK 2024-04-26T15:30:01.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 51306 17.23 MB/sec warmup 85 sec latency 28.951 ms 2024-04-26T15:30:02.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 52100 17.23 MB/sec warmup 86 sec latency 22.460 ms 2024-04-26T15:30:03.078 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:02 smithi112 ceph-mon[21780]: pgmap v285: 97 pgs: 97 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 641 op/s 2024-04-26T15:30:03.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:02 smithi164 ceph-mon[24045]: pgmap v285: 97 pgs: 97 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 641 op/s 2024-04-26T15:30:03.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 52668 17.30 MB/sec warmup 87 sec latency 48.635 ms 2024-04-26T15:30:04.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 53275 17.34 MB/sec warmup 88 sec latency 26.076 ms 2024-04-26T15:30:05.101 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:04 smithi112 ceph-mon[21780]: pgmap v286: 97 pgs: 97 active+clean; 285 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.2 MiB/s wr, 873 op/s 2024-04-26T15:30:05.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:04 smithi164 ceph-mon[24045]: pgmap v286: 97 pgs: 97 active+clean; 285 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.2 MiB/s wr, 873 op/s 2024-04-26T15:30:05.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 53690 17.29 MB/sec warmup 89 sec latency 40.673 ms 2024-04-26T15:30:06.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 54146 17.17 MB/sec warmup 90 sec latency 54.120 ms 2024-04-26T15:30:06.866 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:06 smithi112 ceph-mon[21780]: pgmap v287: 97 pgs: 97 active+clean; 284 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 766 op/s 2024-04-26T15:30:07.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:06 smithi164 ceph-mon[24045]: pgmap v287: 97 pgs: 97 active+clean; 284 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 766 op/s 2024-04-26T15:30:07.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 54641 17.10 MB/sec warmup 91 sec latency 45.268 ms 2024-04-26T15:30:08.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 55239 17.08 MB/sec warmup 92 sec latency 37.980 ms 2024-04-26T15:30:09.057 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:08 smithi112 ceph-mon[21780]: pgmap v288: 97 pgs: 97 active+clean; 289 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.8 MiB/s wr, 884 op/s 2024-04-26T15:30:09.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:08 smithi164 ceph-mon[24045]: pgmap v288: 97 pgs: 97 active+clean; 289 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.8 MiB/s wr, 884 op/s 2024-04-26T15:30:09.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 55795 17.00 MB/sec warmup 93 sec latency 29.654 ms 2024-04-26T15:30:10.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 56242 17.03 MB/sec warmup 94 sec latency 33.627 ms 2024-04-26T15:30:10.870 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:30:11.098 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:10 smithi112 ceph-mon[21780]: pgmap v289: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.9 MiB/s wr, 681 op/s 2024-04-26T15:30:11.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:10 smithi164 ceph-mon[24045]: pgmap v289: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.9 MiB/s wr, 681 op/s 2024-04-26T15:30:11.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 56694 17.02 MB/sec warmup 95 sec latency 60.625 ms 2024-04-26T15:30:11.877 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:30:11.877 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (6m) 96s ago 8m 25.2M - ba2b418f427c c0d9c4d60bab 2024-04-26T15:30:11.877 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (8m) 96s ago 8m 6405k - 17.2.0 e1d6a67b021e c91aa1382673 2024-04-26T15:30:11.877 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (7m) 98s ago 7m 6400k - 17.2.0 e1d6a67b021e ed58e2be875d 2024-04-26T15:30:11.877 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (6m) 96s ago 7m 52.4M - 8.3.5 dad864ee21e9 16304abfa866 2024-04-26T15:30:11.877 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (2m) 96s ago 2m 13.2M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:30:11.877 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (2m) 98s ago 2m 12.0M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:30:11.877 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:9283 running (9m) 96s ago 9m 463M - 17.2.0 e1d6a67b021e 37fa85fc1610 2024-04-26T15:30:11.877 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283 running (7m) 98s ago 7m 414M - 17.2.0 e1d6a67b021e ab46b47c5040 2024-04-26T15:30:11.877 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (9m) 96s ago 9m 43.3M 2048M 17.2.0 e1d6a67b021e 44d7d076db9f 2024-04-26T15:30:11.878 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (7m) 98s ago 7m 38.1M 2048M 17.2.0 e1d6a67b021e 3e800cb663ea 2024-04-26T15:30:11.878 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (107s) 96s ago 107s 51.9M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:30:11.878 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (102s) 98s ago 101s 50.7M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:30:11.878 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (7m) 96s ago 7m 15.3M - 1dbe0e931976 7a4f240204de 2024-04-26T15:30:11.878 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (7m) 98s ago 7m 16.8M - 1dbe0e931976 0b5fa6213ccc 2024-04-26T15:30:11.878 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (6m) 96s ago 6m 42.6M 3216M 17.2.0 e1d6a67b021e 72c22bffb3c9 2024-04-26T15:30:11.878 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (6m) 96s ago 6m 42.8M 3216M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:30:11.878 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (5m) 96s ago 5m 43.3M 3216M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:30:11.878 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (5m) 96s ago 5m 42.8M 3216M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:30:11.878 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (4m) 98s ago 4m 43.4M 3984M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:30:11.878 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (4m) 98s ago 4m 39.1M 3984M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:30:11.878 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (4m) 98s ago 4m 40.1M 3984M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:30:11.879 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (3m) 98s ago 3m 44.0M 3984M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:30:11.879 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (6m) 96s ago 6m 63.1M - 514e6a882f6e ce3b8a2da81f 2024-04-26T15:30:12.110 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:11 smithi112 ceph-mon[21780]: from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:12.342 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:30:12.342 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:30:12.342 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:30:12.342 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:30:12.342 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:30:12.342 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:30:12.342 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:30:12.342 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:30:12.343 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T15:30:12.343 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:30:12.343 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:30:12.343 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:30:12.343 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:30:12.343 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:30:12.343 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-26T15:30:12.343 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:30:12.343 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:30:12.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:11 smithi164 ceph-mon[24045]: from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:12.724 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:30:12.724 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:30:12.724 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:30:12.724 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [], 2024-04-26T15:30:12.724 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "0/23 daemons upgraded", 2024-04-26T15:30:12.724 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 image on host smithi164" 2024-04-26T15:30:12.724 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:30:12.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 57070 16.99 MB/sec warmup 96 sec latency 59.439 ms 2024-04-26T15:30:12.865 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:12 smithi112 ceph-mon[21780]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:12.865 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:12 smithi112 ceph-mon[21780]: pgmap v290: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.1 MiB/s wr, 749 op/s 2024-04-26T15:30:12.865 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:12 smithi112 ceph-mon[21780]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:12.865 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:12 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1279405212' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:30:13.193 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:30:13.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:12 smithi164 ceph-mon[24045]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:13.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:12 smithi164 ceph-mon[24045]: pgmap v290: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.1 MiB/s wr, 749 op/s 2024-04-26T15:30:13.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:12 smithi164 ceph-mon[24045]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:13.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:12 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1279405212' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:30:13.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 57343 16.87 MB/sec warmup 97 sec latency 36.081 ms 2024-04-26T15:30:14.138 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:13 smithi112 ceph-mon[21780]: from='client.14668 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:14.139 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:13 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1458886437' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:30:14.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:13 smithi164 ceph-mon[24045]: from='client.14668 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:14.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:13 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1458886437' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:30:14.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 57668 16.74 MB/sec warmup 98 sec latency 49.088 ms 2024-04-26T15:30:14.912 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:14 smithi112 ceph-mon[21780]: pgmap v291: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 921 op/s 2024-04-26T15:30:15.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:14 smithi164 ceph-mon[24045]: pgmap v291: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 921 op/s 2024-04-26T15:30:15.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 58067 16.63 MB/sec warmup 99 sec latency 83.949 ms 2024-04-26T15:30:16.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 58557 16.60 MB/sec warmup 100 sec latency 47.642 ms 2024-04-26T15:30:16.928 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:16 smithi112 ceph-mon[21780]: pgmap v292: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.8 MiB/s wr, 586 op/s 2024-04-26T15:30:17.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:16 smithi164 ceph-mon[24045]: pgmap v292: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.8 MiB/s wr, 586 op/s 2024-04-26T15:30:17.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 59245 16.58 MB/sec warmup 101 sec latency 26.286 ms 2024-04-26T15:30:18.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 60033 16.71 MB/sec warmup 102 sec latency 27.811 ms 2024-04-26T15:30:18.947 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:18 smithi112 ceph-mon[21780]: pgmap v293: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.3 MiB/s wr, 693 op/s 2024-04-26T15:30:19.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:18 smithi164 ceph-mon[24045]: pgmap v293: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.3 MiB/s wr, 693 op/s 2024-04-26T15:30:19.809 INFO:teuthology.orchestra.run.smithi112.stdout: 5 60634 16.77 MB/sec warmup 103 sec latency 31.947 ms 2024-04-26T15:30:20.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:20 smithi164 ceph-mon[24045]: Upgrade: Updating mgr.smithi164.tuhwus 2024-04-26T15:30:20.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:20.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.tuhwus", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:30:20.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:30:20.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:20 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:30:20.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:20 smithi164 ceph-mon[24045]: Deploying daemon mgr.smithi164.tuhwus on smithi164 2024-04-26T15:30:20.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:20 smithi112 ceph-mon[21780]: Upgrade: Updating mgr.smithi164.tuhwus 2024-04-26T15:30:20.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:20.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.tuhwus", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:30:20.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:30:20.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:20 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:30:20.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:20 smithi112 ceph-mon[21780]: Deploying daemon mgr.smithi164.tuhwus on smithi164 2024-04-26T15:30:20.816 INFO:teuthology.orchestra.run.smithi112.stdout: 5 61219 16.70 MB/sec warmup 104 sec latency 24.499 ms 2024-04-26T15:30:21.498 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:21 smithi112 ceph-mon[21780]: pgmap v294: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.8 MiB/s wr, 656 op/s 2024-04-26T15:30:21.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:21 smithi164 ceph-mon[24045]: pgmap v294: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.8 MiB/s wr, 656 op/s 2024-04-26T15:30:21.816 INFO:teuthology.orchestra.run.smithi112.stdout: 5 61663 16.61 MB/sec warmup 105 sec latency 47.663 ms 2024-04-26T15:30:22.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 62302 16.63 MB/sec warmup 106 sec latency 20.376 ms 2024-04-26T15:30:23.517 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:23 smithi112 ceph-mon[21780]: pgmap v295: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.4 MiB/s wr, 613 op/s 2024-04-26T15:30:23.564 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:23 smithi164 ceph-mon[24045]: pgmap v295: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.4 MiB/s wr, 613 op/s 2024-04-26T15:30:23.816 INFO:teuthology.orchestra.run.smithi112.stdout: 5 62853 16.55 MB/sec warmup 107 sec latency 36.234 ms 2024-04-26T15:30:24.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 63376 16.60 MB/sec warmup 108 sec latency 77.310 ms 2024-04-26T15:30:25.335 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:25 smithi112 ceph-mon[21780]: pgmap v296: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.1 MiB/s wr, 776 op/s 2024-04-26T15:30:25.464 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:25 smithi164 ceph-mon[24045]: pgmap v296: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.1 MiB/s wr, 776 op/s 2024-04-26T15:30:25.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 63941 16.63 MB/sec warmup 109 sec latency 26.677 ms 2024-04-26T15:30:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:26 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:26.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:26 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:30:26.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:26 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:30:26.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:26 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:30:26.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:26 smithi164 ceph-mon[24045]: pgmap v297: 97 pgs: 97 active+clean; 323 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.7 MiB/s wr, 681 op/s 2024-04-26T15:30:26.601 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:26 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:26.601 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:26 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:30:26.601 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:26 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:30:26.601 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:26 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:30:26.602 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:26 smithi112 ceph-mon[21780]: pgmap v297: 97 pgs: 97 active+clean; 323 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.7 MiB/s wr, 681 op/s 2024-04-26T15:30:26.816 INFO:teuthology.orchestra.run.smithi112.stdout: 5 64334 16.60 MB/sec warmup 110 sec latency 33.660 ms 2024-04-26T15:30:27.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 64748 16.50 MB/sec warmup 111 sec latency 39.597 ms 2024-04-26T15:30:27.862 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:27 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:30:27.863 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:27 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:30:28.027 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:27 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:30:28.027 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:27 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:30:28.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 65220 16.45 MB/sec warmup 112 sec latency 21.384 ms 2024-04-26T15:30:28.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:28 smithi164 ceph-mon[24045]: pgmap v298: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.1 MiB/s wr, 847 op/s 2024-04-26T15:30:28.872 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:28 smithi112 ceph-mon[21780]: pgmap v298: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.1 MiB/s wr, 847 op/s 2024-04-26T15:30:29.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 65644 16.38 MB/sec warmup 113 sec latency 116.230 ms 2024-04-26T15:30:30.386 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:30 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:30.387 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:30 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:30.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:30 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:30.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:30 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:30.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 66227 16.34 MB/sec warmup 114 sec latency 56.055 ms 2024-04-26T15:30:31.360 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:31 smithi112 ceph-mon[21780]: pgmap v299: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.1 MiB/s wr, 756 op/s 2024-04-26T15:30:31.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:31 smithi164 ceph-mon[24045]: pgmap v299: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.1 MiB/s wr, 756 op/s 2024-04-26T15:30:31.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 66639 16.32 MB/sec warmup 115 sec latency 53.215 ms 2024-04-26T15:30:32.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 67013 16.31 MB/sec warmup 116 sec latency 29.827 ms 2024-04-26T15:30:33.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:33 smithi164 ceph-mon[24045]: pgmap v300: 97 pgs: 97 active+clean; 333 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.8 MiB/s wr, 719 op/s 2024-04-26T15:30:33.389 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:33 smithi112 ceph-mon[21780]: pgmap v300: 97 pgs: 97 active+clean; 333 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.8 MiB/s wr, 719 op/s 2024-04-26T15:30:33.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 67390 16.27 MB/sec warmup 117 sec latency 144.214 ms 2024-04-26T15:30:34.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 67752 16.26 MB/sec warmup 118 sec latency 199.931 ms 2024-04-26T15:30:35.166 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:35 smithi112 ceph-mon[21780]: pgmap v301: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 990 op/s 2024-04-26T15:30:35.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:35 smithi112 ceph-mon[21780]: Standby manager daemon smithi164.tuhwus restarted 2024-04-26T15:30:35.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:35 smithi112 ceph-mon[21780]: Standby manager daemon smithi164.tuhwus started 2024-04-26T15:30:35.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:35 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/crt"}]: dispatch 2024-04-26T15:30:35.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:35 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:30:35.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:35 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/key"}]: dispatch 2024-04-26T15:30:35.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:35 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:30:35.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:35 smithi164 ceph-mon[24045]: pgmap v301: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 990 op/s 2024-04-26T15:30:35.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:35 smithi164 ceph-mon[24045]: Standby manager daemon smithi164.tuhwus restarted 2024-04-26T15:30:35.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:35 smithi164 ceph-mon[24045]: Standby manager daemon smithi164.tuhwus started 2024-04-26T15:30:35.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:35 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/crt"}]: dispatch 2024-04-26T15:30:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:35 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:30:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:35 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/key"}]: dispatch 2024-04-26T15:30:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:35 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:30:35.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 67998 16.16 MB/sec warmup 119 sec latency 56.299 ms 2024-04-26T15:30:36.193 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:36 smithi112 ceph-mon[21780]: mgrmap e22: smithi112.llioug(active, since 8m), standbys: smithi164.tuhwus 2024-04-26T15:30:36.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:36 smithi164 ceph-mon[24045]: mgrmap e22: smithi112.llioug(active, since 8m), standbys: smithi164.tuhwus 2024-04-26T15:30:37.207 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:37 smithi112 ceph-mon[21780]: pgmap v302: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.5 MiB/s wr, 726 op/s 2024-04-26T15:30:37.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:37 smithi164 ceph-mon[24045]: pgmap v302: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.5 MiB/s wr, 726 op/s 2024-04-26T15:30:37.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 68903 10.13 MB/sec execute 1 sec latency 47.426 ms 2024-04-26T15:30:38.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 69571 14.07 MB/sec execute 2 sec latency 26.036 ms 2024-04-26T15:30:39.224 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:39 smithi112 ceph-mon[21780]: pgmap v303: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 667 op/s 2024-04-26T15:30:39.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:39 smithi164 ceph-mon[24045]: pgmap v303: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 667 op/s 2024-04-26T15:30:39.818 INFO:teuthology.orchestra.run.smithi112.stdout: 5 70180 14.51 MB/sec execute 3 sec latency 31.196 ms 2024-04-26T15:30:40.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 70762 16.89 MB/sec execute 4 sec latency 21.821 ms 2024-04-26T15:30:41.275 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:41 smithi112 ceph-mon[21780]: pgmap v304: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.0 MiB/s wr, 608 op/s 2024-04-26T15:30:41.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:41 smithi164 ceph-mon[24045]: pgmap v304: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.0 MiB/s wr, 608 op/s 2024-04-26T15:30:41.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 71223 16.48 MB/sec execute 5 sec latency 58.324 ms 2024-04-26T15:30:42.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 71583 15.07 MB/sec execute 6 sec latency 46.286 ms 2024-04-26T15:30:43.282 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:43 smithi164 ceph-mon[24045]: pgmap v305: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 608 op/s 2024-04-26T15:30:43.296 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:43 smithi112 ceph-mon[21780]: pgmap v305: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 608 op/s 2024-04-26T15:30:43.619 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:30:43.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 72055 13.90 MB/sec execute 7 sec latency 30.391 ms 2024-04-26T15:30:44.420 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:30:44.420 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (7m) 2m ago 8m 25.2M - ba2b418f427c c0d9c4d60bab 2024-04-26T15:30:44.420 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (8m) 2m ago 8m 6405k - 17.2.0 e1d6a67b021e c91aa1382673 2024-04-26T15:30:44.420 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (8m) 15s ago 8m 6400k - 17.2.0 e1d6a67b021e ed58e2be875d 2024-04-26T15:30:44.420 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (6m) 2m ago 8m 52.4M - 8.3.5 dad864ee21e9 16304abfa866 2024-04-26T15:30:44.420 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (2m) 2m ago 2m 13.2M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:30:44.420 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (2m) 15s ago 2m 12.4M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:30:44.420 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:9283 running (9m) 2m ago 9m 463M - 17.2.0 e1d6a67b021e 37fa85fc1610 2024-04-26T15:30:44.420 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283 running (19s) 15s ago 7m 42.3M - 19.0.0-3305-ga430f9ed 031ba2747cfd b621bf362ce7 2024-04-26T15:30:44.420 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (9m) 2m ago 10m 43.3M 2048M 17.2.0 e1d6a67b021e 44d7d076db9f 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (7m) 15s ago 7m 38.8M 2048M 17.2.0 e1d6a67b021e 3e800cb663ea 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (2m) 2m ago 2m 51.9M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (2m) 15s ago 2m 74.9M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (8m) 2m ago 8m 15.3M - 1dbe0e931976 7a4f240204de 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (7m) 15s ago 7m 19.1M - 1dbe0e931976 0b5fa6213ccc 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (7m) 2m ago 7m 42.6M 3216M 17.2.0 e1d6a67b021e 72c22bffb3c9 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (6m) 2m ago 6m 42.8M 3216M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (6m) 2m ago 6m 43.3M 3216M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (5m) 2m ago 5m 42.8M 3216M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (5m) 15s ago 5m 246M 3984M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (4m) 15s ago 4m 275M 3984M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (4m) 15s ago 4m 243M 3984M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (4m) 15s ago 4m 337M 3984M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:30:44.421 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (7m) 2m ago 7m 63.1M - 514e6a882f6e ce3b8a2da81f 2024-04-26T15:30:44.560 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:44.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:44.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:44.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:30:44.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:44 smithi112 ceph-mon[21780]: Upgrade: Need to upgrade myself (mgr.smithi112.llioug) 2024-04-26T15:30:44.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:44 smithi112 ceph-mon[21780]: Failing over to other MGR 2024-04-26T15:30:44.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:44 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr fail", "who": "smithi112.llioug"}]: dispatch 2024-04-26T15:30:44.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:44 smithi112 ceph-mon[21780]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T15:30:44.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:44 smithi112 ceph-mon[21780]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:44.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:44 smithi112 ceph-mon[21780]: pgmap v307: 97 pgs: 97 active+clean; 334 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.0 MiB/s wr, 652 op/s 2024-04-26T15:30:44.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:44 smithi112 ceph-mon[21780]: from='client.14684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:44.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:44.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:44.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' 2024-04-26T15:30:44.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:30:44.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:44 smithi164 ceph-mon[24045]: Upgrade: Need to upgrade myself (mgr.smithi112.llioug) 2024-04-26T15:30:44.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:44 smithi164 ceph-mon[24045]: Failing over to other MGR 2024-04-26T15:30:44.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:44 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr fail", "who": "smithi112.llioug"}]: dispatch 2024-04-26T15:30:44.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:44 smithi164 ceph-mon[24045]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T15:30:44.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:44 smithi164 ceph-mon[24045]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:44.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:44 smithi164 ceph-mon[24045]: pgmap v307: 97 pgs: 97 active+clean; 334 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.0 MiB/s wr, 652 op/s 2024-04-26T15:30:44.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:44 smithi164 ceph-mon[24045]: from='client.14684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:44.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 72469 13.23 MB/sec execute 8 sec latency 48.705 ms 2024-04-26T15:30:45.042 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:30:45.042 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 1 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-26T15:30:45.043 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 1 2024-04-26T15:30:45.044 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:30:45.044 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:30:45.569 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:30:45.569 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:30:45.569 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:30:45.569 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:30:45.569 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [], 2024-04-26T15:30:45.569 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "1/23 daemons upgraded", 2024-04-26T15:30:45.569 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "", 2024-04-26T15:30:45.569 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:30:45.570 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:30:45.578 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "mgr fail", "who": "smithi112.llioug"}]': finished 2024-04-26T15:30:45.578 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: mgrmap e23: smithi164.tuhwus(active, starting, since 0.99826s) 2024-04-26T15:30:45.578 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:30:45.578 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:30:45.578 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:30:45.578 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.gbxewm"}]: dispatch 2024-04-26T15:30:45.578 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mgr metadata", "who": "smithi164.tuhwus", "id": "smithi164.tuhwus"}]: dispatch 2024-04-26T15:30:45.578 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:30:45.578 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: Manager daemon smithi164.tuhwus is now available 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.tuhwus/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.tuhwus/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:30:45.579 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.tuhwus/trash_purge_schedule"}]: dispatch 2024-04-26T15:30:45.580 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.tuhwus/trash_purge_schedule"}]: dispatch 2024-04-26T15:30:45.580 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1231663934' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:30:45.580 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: Deploying cephadm binary to smithi164 2024-04-26T15:30:45.580 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:45 smithi112 ceph-mon[21780]: Deploying cephadm binary to smithi112 2024-04-26T15:30:45.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 73011 13.57 MB/sec execute 9 sec latency 41.093 ms 2024-04-26T15:30:45.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14202 172.21.15.112:0/92923101' entity='mgr.smithi112.llioug' cmd='[{"prefix": "mgr fail", "who": "smithi112.llioug"}]': finished 2024-04-26T15:30:45.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: mgrmap e23: smithi164.tuhwus(active, starting, since 0.99826s) 2024-04-26T15:30:45.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:30:45.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:30:45.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:30:45.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.gbxewm"}]: dispatch 2024-04-26T15:30:45.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mgr metadata", "who": "smithi164.tuhwus", "id": "smithi164.tuhwus"}]: dispatch 2024-04-26T15:30:45.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:30:45.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: Manager daemon smithi164.tuhwus is now available 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:30:45.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:30:45.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.tuhwus/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:30:45.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.tuhwus/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:30:45.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.tuhwus/trash_purge_schedule"}]: dispatch 2024-04-26T15:30:45.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.tuhwus/trash_purge_schedule"}]: dispatch 2024-04-26T15:30:45.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1231663934' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:30:45.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: Deploying cephadm binary to smithi164 2024-04-26T15:30:45.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:45 smithi164 ceph-mon[24045]: Deploying cephadm binary to smithi112 2024-04-26T15:30:46.319 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:30:46.583 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:46 smithi112 ceph-mon[21780]: mgrmap e24: smithi164.tuhwus(active, since 2s) 2024-04-26T15:30:46.584 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:46 smithi112 ceph-mon[21780]: pgmap v3: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:30:46.584 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:46 smithi112 ceph-mon[21780]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:46.584 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:46 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/202186971' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:30:46.584 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:46 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:46.584 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:46 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:46.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 73660 13.72 MB/sec execute 10 sec latency 30.236 ms 2024-04-26T15:30:46.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:46 smithi164 ceph-mon[24045]: mgrmap e24: smithi164.tuhwus(active, since 2s) 2024-04-26T15:30:46.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:46 smithi164 ceph-mon[24045]: pgmap v3: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:30:46.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:46 smithi164 ceph-mon[24045]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:30:46.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:46 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/202186971' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:30:46.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:46 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:46.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:46 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:47.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 74240 14.44 MB/sec execute 11 sec latency 21.850 ms 2024-04-26T15:30:47.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:47 smithi112 ceph-mon[21780]: pgmap v4: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:30:47.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:47 smithi112 ceph-mon[21780]: [26/Apr/2024:15:30:46] ENGINE Bus STARTING 2024-04-26T15:30:47.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:47 smithi112 ceph-mon[21780]: mgrmap e25: smithi164.tuhwus(active, since 3s) 2024-04-26T15:30:47.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:47 smithi112 ceph-mon[21780]: [26/Apr/2024:15:30:46] ENGINE Serving on https://172.21.15.164:7150 2024-04-26T15:30:47.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:47 smithi112 ceph-mon[21780]: [26/Apr/2024:15:30:47] ENGINE Serving on http://172.21.15.164:8765 2024-04-26T15:30:47.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:47 smithi112 ceph-mon[21780]: [26/Apr/2024:15:30:47] ENGINE Bus STARTED 2024-04-26T15:30:48.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:47 smithi164 ceph-mon[24045]: pgmap v4: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:30:48.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:47 smithi164 ceph-mon[24045]: [26/Apr/2024:15:30:46] ENGINE Bus STARTING 2024-04-26T15:30:48.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:47 smithi164 ceph-mon[24045]: mgrmap e25: smithi164.tuhwus(active, since 3s) 2024-04-26T15:30:48.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:47 smithi164 ceph-mon[24045]: [26/Apr/2024:15:30:46] ENGINE Serving on https://172.21.15.164:7150 2024-04-26T15:30:48.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:47 smithi164 ceph-mon[24045]: [26/Apr/2024:15:30:47] ENGINE Serving on http://172.21.15.164:8765 2024-04-26T15:30:48.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:47 smithi164 ceph-mon[24045]: [26/Apr/2024:15:30:47] ENGINE Bus STARTED 2024-04-26T15:30:48.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 74727 14.64 MB/sec execute 12 sec latency 28.897 ms 2024-04-26T15:30:49.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 75048 14.19 MB/sec execute 13 sec latency 29.427 ms 2024-04-26T15:30:49.865 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:49 smithi112 ceph-mon[21780]: pgmap v5: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:30:49.865 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:49 smithi112 ceph-mon[21780]: mgrmap e26: smithi164.tuhwus(active, since 5s) 2024-04-26T15:30:50.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:49 smithi164 ceph-mon[24045]: pgmap v5: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:30:50.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:49 smithi164 ceph-mon[24045]: mgrmap e26: smithi164.tuhwus(active, since 5s) 2024-04-26T15:30:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 75430 13.64 MB/sec execute 14 sec latency 32.785 ms 2024-04-26T15:30:51.817 INFO:teuthology.orchestra.run.smithi112.stdout: 5 75837 13.27 MB/sec execute 15 sec latency 45.437 ms 2024-04-26T15:30:51.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:51 smithi164 ceph-mon[24045]: pgmap v6: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:30:51.880 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:51 smithi112 ceph-mon[21780]: pgmap v6: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:30:52.818 INFO:teuthology.orchestra.run.smithi112.stdout: 5 76320 13.25 MB/sec execute 16 sec latency 28.561 ms 2024-04-26T15:30:53.818 INFO:teuthology.orchestra.run.smithi112.stdout: 5 76965 13.19 MB/sec execute 17 sec latency 34.760 ms 2024-04-26T15:30:53.896 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:53 smithi112 ceph-mon[21780]: pgmap v7: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 5.7 MiB/s wr, 515 op/s 2024-04-26T15:30:53.896 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:53 smithi112 ceph-mon[21780]: Standby manager daemon smithi112.llioug started 2024-04-26T15:30:53.896 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:53 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.112:0/860315966' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.llioug/crt"}]: dispatch 2024-04-26T15:30:53.896 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:53 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.112:0/860315966' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:30:53.896 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:53 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.112:0/860315966' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.llioug/key"}]: dispatch 2024-04-26T15:30:53.896 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:53 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.112:0/860315966' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:30:54.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:53 smithi164 ceph-mon[24045]: pgmap v7: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 5.7 MiB/s wr, 515 op/s 2024-04-26T15:30:54.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:53 smithi164 ceph-mon[24045]: Standby manager daemon smithi112.llioug started 2024-04-26T15:30:54.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:53 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.112:0/860315966' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.llioug/crt"}]: dispatch 2024-04-26T15:30:54.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:53 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.112:0/860315966' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:30:54.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:53 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.112:0/860315966' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.llioug/key"}]: dispatch 2024-04-26T15:30:54.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:53 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.112:0/860315966' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:30:54.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 77511 13.53 MB/sec execute 18 sec latency 42.521 ms 2024-04-26T15:30:54.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:54 smithi112 ceph-mon[21780]: mgrmap e27: smithi164.tuhwus(active, since 10s), standbys: smithi112.llioug 2024-04-26T15:30:54.903 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:54 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mgr metadata", "who": "smithi112.llioug", "id": "smithi112.llioug"}]: dispatch 2024-04-26T15:30:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:54 smithi164 ceph-mon[24045]: mgrmap e27: smithi164.tuhwus(active, since 10s), standbys: smithi112.llioug 2024-04-26T15:30:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:54 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mgr metadata", "who": "smithi112.llioug", "id": "smithi112.llioug"}]: dispatch 2024-04-26T15:30:55.818 INFO:teuthology.orchestra.run.smithi112.stdout: 5 78110 14.04 MB/sec execute 19 sec latency 27.937 ms 2024-04-26T15:30:56.098 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:55 smithi112 ceph-mon[21780]: pgmap v8: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 4.4 MiB/s wr, 400 op/s 2024-04-26T15:30:56.099 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:55 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:56.099 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:55 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:56.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:55 smithi164 ceph-mon[24045]: pgmap v8: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 4.4 MiB/s wr, 400 op/s 2024-04-26T15:30:56.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:55 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:56.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:55 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:56.818 INFO:teuthology.orchestra.run.smithi112.stdout: 5 78595 14.00 MB/sec execute 20 sec latency 45.118 ms 2024-04-26T15:30:57.102 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:57 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:57.102 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:57 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:57.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:57 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:57.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:57 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:30:57.818 INFO:teuthology.orchestra.run.smithi112.stdout: 5 79045 13.71 MB/sec execute 21 sec latency 45.908 ms 2024-04-26T15:30:58.362 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:30:58 smithi112 ceph-mon[21780]: pgmap v9: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 7.0 MiB/s wr, 663 op/s 2024-04-26T15:30:58.499 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:30:58 smithi164 ceph-mon[24045]: pgmap v9: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 7.0 MiB/s wr, 663 op/s 2024-04-26T15:30:58.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 79467 13.47 MB/sec execute 22 sec latency 37.782 ms 2024-04-26T15:30:59.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 79984 13.50 MB/sec execute 23 sec latency 41.401 ms 2024-04-26T15:31:00.135 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:00 smithi112 ceph-mon[21780]: pgmap v10: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-04-26T15:31:00.135 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:00 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:00.135 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:00 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:00.136 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:00 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:31:00.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:00 smithi164 ceph-mon[24045]: pgmap v10: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-04-26T15:31:00.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:00 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:00.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:00 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:00.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:00 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:31:00.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 80465 13.32 MB/sec execute 24 sec latency 39.389 ms 2024-04-26T15:31:01.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: pgmap v11: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-04-26T15:31:01.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:01.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:01.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: Adjusting osd_memory_target on smithi164 to 2704M 2024-04-26T15:31:01.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:01.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:01.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:01.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: Adjusting osd_memory_target on smithi112 to 1936M 2024-04-26T15:31:01.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:01.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:31:01.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:31:01.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-26T15:31:01.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:01 smithi112 ceph-mon[21780]: Updating smithi164:/etc/ceph/ceph.conf 2024-04-26T15:31:01.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 80921 13.33 MB/sec execute 25 sec latency 33.341 ms 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: pgmap v11: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: Adjusting osd_memory_target on smithi164 to 2704M 2024-04-26T15:31:01.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: Adjusting osd_memory_target on smithi112 to 1936M 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:01.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:31:01.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:31:01.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-26T15:31:01.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:01 smithi164 ceph-mon[24045]: Updating smithi164:/etc/ceph/ceph.conf 2024-04-26T15:31:02.689 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:02 smithi112 ceph-mon[21780]: Updating smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T15:31:02.689 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:02 smithi112 ceph-mon[21780]: Updating smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T15:31:02.689 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:02 smithi112 ceph-mon[21780]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:31:02.689 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:02 smithi112 ceph-mon[21780]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:31:02.689 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:02 smithi112 ceph-mon[21780]: Updating smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T15:31:02.689 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:02 smithi112 ceph-mon[21780]: Updating smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T15:31:02.689 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:02 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:02.689 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:02 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:02.689 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:02 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:02.689 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:02 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:02.689 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:02 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:02.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 81403 13.50 MB/sec execute 26 sec latency 41.034 ms 2024-04-26T15:31:02.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:02 smithi164 ceph-mon[24045]: Updating smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T15:31:02.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:02 smithi164 ceph-mon[24045]: Updating smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T15:31:02.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:02 smithi164 ceph-mon[24045]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:31:02.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:02 smithi164 ceph-mon[24045]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:31:02.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:02 smithi164 ceph-mon[24045]: Updating smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T15:31:02.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:02 smithi164 ceph-mon[24045]: Updating smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T15:31:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:02 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:02 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:02 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:02 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:02 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:03.703 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:03 smithi112 ceph-mon[21780]: pgmap v12: 97 pgs: 97 active+clean; 325 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 9.1 MiB/s wr, 897 op/s 2024-04-26T15:31:03.704 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:03 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:03.704 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:03 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:03.704 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:03 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:03.704 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:03 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:03.704 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:03 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:03.704 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:03 smithi112 ceph-mon[21780]: Reconfiguring alertmanager.smithi112 (dependencies changed)... 2024-04-26T15:31:03.704 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:03 smithi112 ceph-mon[21780]: Reconfiguring daemon alertmanager.smithi112 on smithi112 2024-04-26T15:31:03.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 81830 13.59 MB/sec execute 27 sec latency 41.794 ms 2024-04-26T15:31:03.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:03 smithi164 ceph-mon[24045]: pgmap v12: 97 pgs: 97 active+clean; 325 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 9.1 MiB/s wr, 897 op/s 2024-04-26T15:31:03.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:03 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:03.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:03 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:03.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:03 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:03.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:03 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:03.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:03 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:03.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:03 smithi164 ceph-mon[24045]: Reconfiguring alertmanager.smithi112 (dependencies changed)... 2024-04-26T15:31:03.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:03 smithi164 ceph-mon[24045]: Reconfiguring daemon alertmanager.smithi112 on smithi112 2024-04-26T15:31:04.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 82232 13.45 MB/sec execute 28 sec latency 59.607 ms 2024-04-26T15:31:05.741 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:05 smithi112 ceph-mon[21780]: pgmap v13: 97 pgs: 97 active+clean; 325 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.8 MiB/s wr, 599 op/s 2024-04-26T15:31:05.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 82734 13.26 MB/sec execute 29 sec latency 41.952 ms 2024-04-26T15:31:05.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:05 smithi164 ceph-mon[24045]: pgmap v13: 97 pgs: 97 active+clean; 325 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.8 MiB/s wr, 599 op/s 2024-04-26T15:31:06.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 83332 13.33 MB/sec execute 30 sec latency 29.810 ms 2024-04-26T15:31:07.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 84002 13.38 MB/sec execute 31 sec latency 37.708 ms 2024-04-26T15:31:08.018 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:07 smithi112 ceph-mon[21780]: pgmap v14: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 866 op/s 2024-04-26T15:31:08.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:07 smithi164 ceph-mon[24045]: pgmap v14: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 866 op/s 2024-04-26T15:31:08.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 84656 13.68 MB/sec execute 32 sec latency 42.572 ms 2024-04-26T15:31:09.788 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:09 smithi112 ceph-mon[21780]: pgmap v15: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 559 op/s 2024-04-26T15:31:09.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 85183 13.84 MB/sec execute 33 sec latency 29.247 ms 2024-04-26T15:31:10.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:09 smithi164 ceph-mon[24045]: pgmap v15: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 559 op/s 2024-04-26T15:31:10.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 85611 13.76 MB/sec execute 34 sec latency 34.341 ms 2024-04-26T15:31:11.816 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:11 smithi112 ceph-mon[21780]: pgmap v16: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 559 op/s 2024-04-26T15:31:11.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 86088 13.67 MB/sec execute 35 sec latency 30.134 ms 2024-04-26T15:31:12.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:11 smithi164 ceph-mon[24045]: pgmap v16: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 559 op/s 2024-04-26T15:31:12.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 86533 13.53 MB/sec execute 36 sec latency 25.732 ms 2024-04-26T15:31:13.082 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:12 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:13.082 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:12 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:13.082 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:12 smithi112 ceph-mon[21780]: Reconfiguring grafana.smithi112 (dependencies changed)... 2024-04-26T15:31:13.082 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:12 smithi112 ceph-mon[21780]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-26T15:31:13.082 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:12 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:13.082 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:12 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:13.082 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:12 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T15:31:13.082 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:12 smithi112 ceph-mon[21780]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T15:31:13.082 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:12 smithi112 ceph-mon[21780]: Reconfiguring daemon grafana.smithi112 on smithi112 2024-04-26T15:31:13.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:12 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:13.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:12 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:13.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:12 smithi164 ceph-mon[24045]: Reconfiguring grafana.smithi112 (dependencies changed)... 2024-04-26T15:31:13.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:12 smithi164 ceph-mon[24045]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-26T15:31:13.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:12 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:13.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:12 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:13.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:12 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T15:31:13.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:12 smithi164 ceph-mon[24045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T15:31:13.281 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:12 smithi164 ceph-mon[24045]: Reconfiguring daemon grafana.smithi112 on smithi112 2024-04-26T15:31:13.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 87065 13.56 MB/sec execute 37 sec latency 31.107 ms 2024-04-26T15:31:14.350 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:14 smithi112 ceph-mon[21780]: pgmap v17: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 901 op/s 2024-04-26T15:31:14.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:14 smithi164 ceph-mon[24045]: pgmap v17: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 901 op/s 2024-04-26T15:31:14.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 87611 13.54 MB/sec execute 38 sec latency 31.845 ms 2024-04-26T15:31:15.368 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:15 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:31:15.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:15 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:31:15.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 88010 13.46 MB/sec execute 39 sec latency 42.977 ms 2024-04-26T15:31:16.389 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:16 smithi112 ceph-mon[21780]: pgmap v18: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.7 MiB/s wr, 608 op/s 2024-04-26T15:31:16.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:16 smithi164 ceph-mon[24045]: pgmap v18: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.7 MiB/s wr, 608 op/s 2024-04-26T15:31:16.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 88476 13.53 MB/sec execute 40 sec latency 32.196 ms 2024-04-26T15:31:16.821 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:31:17.729 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:31:17.729 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 starting - - - - 2024-04-26T15:31:17.729 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (9m) 18s ago 9m 6405k - 17.2.0 e1d6a67b021e c91aa1382673 2024-04-26T15:31:17.729 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (8m) 22s ago 8m 6400k - 17.2.0 e1d6a67b021e ed58e2be875d 2024-04-26T15:31:17.729 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (7m) 18s ago 9m 54.6M - 8.3.5 dad864ee21e9 16304abfa866 2024-04-26T15:31:17.729 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (3m) 18s ago 3m 36.2M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:31:17.729 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (3m) 22s ago 3m 12.8M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:31:17.730 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:9283 running (10m) 18s ago 10m 313M - 17.2.0 e1d6a67b021e 37fa85fc1610 2024-04-26T15:31:17.730 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283 running (52s) 22s ago 8m 505M - 19.0.0-3305-ga430f9ed 031ba2747cfd b621bf362ce7 2024-04-26T15:31:17.730 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (10m) 18s ago 10m 48.5M 2048M 17.2.0 e1d6a67b021e 44d7d076db9f 2024-04-26T15:31:17.730 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (8m) 22s ago 8m 40.5M 2048M 17.2.0 e1d6a67b021e 3e800cb663ea 2024-04-26T15:31:17.730 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (2m) 18s ago 2m 247M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:31:17.730 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (2m) 22s ago 2m 79.0M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:31:17.730 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (8m) 18s ago 8m 19.7M - 1.3.1 1dbe0e931976 7a4f240204de 2024-04-26T15:31:17.730 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (8m) 22s ago 8m 18.9M - 1.3.1 1dbe0e931976 0b5fa6213ccc 2024-04-26T15:31:17.730 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (7m) 18s ago 7m 313M 1936M 17.2.0 e1d6a67b021e 72c22bffb3c9 2024-04-26T15:31:17.730 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (7m) 18s ago 7m 248M 1936M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:31:17.730 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (6m) 18s ago 6m 212M 1936M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:31:17.730 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (6m) 18s ago 6m 263M 1936M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:31:17.731 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (5m) 22s ago 5m 286M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:31:17.731 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (5m) 22s ago 5m 316M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:31:17.731 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (5m) 22s ago 5m 263M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:31:17.731 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (4m) 22s ago 4m 378M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:31:17.731 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (8m) 18s ago 8m 65.4M - 2.33.4 514e6a882f6e ce3b8a2da81f 2024-04-26T15:31:17.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 88914 13.61 MB/sec execute 41 sec latency 34.456 ms 2024-04-26T15:31:18.157 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:18 smithi112 ceph-mon[21780]: pgmap v19: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 900 op/s 2024-04-26T15:31:18.158 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:18 smithi112 ceph-mon[21780]: from='client.14722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:18.415 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:31:18.415 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:31:18.415 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:31:18.415 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:31:18.415 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:31:18.415 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-26T15:31:18.415 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 1 2024-04-26T15:31:18.415 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:31:18.415 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:31:18.415 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T15:31:18.416 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:31:18.416 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:31:18.416 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:31:18.416 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:31:18.416 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:31:18.416 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-26T15:31:18.416 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 1 2024-04-26T15:31:18.416 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:31:18.416 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:31:18.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:18 smithi164 ceph-mon[24045]: pgmap v19: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 900 op/s 2024-04-26T15:31:18.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:18 smithi164 ceph-mon[24045]: from='client.14722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:18.819 INFO:teuthology.orchestra.run.smithi112.stdout: 5 89353 13.59 MB/sec execute 42 sec latency 31.792 ms 2024-04-26T15:31:18.972 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:31:18.972 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:31:18.972 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:31:18.972 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:31:18.972 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [], 2024-04-26T15:31:18.972 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "1/23 daemons upgraded", 2024-04-26T15:31:18.972 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "", 2024-04-26T15:31:18.972 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:31:18.973 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:31:19.283 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:19 smithi112 ceph-mon[21780]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:19.284 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:19 smithi112 ceph-mon[21780]: from='client.14730 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:19.284 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:19 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/39866620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:31:19.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:19 smithi164 ceph-mon[24045]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:19.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:19 smithi164 ceph-mon[24045]: from='client.14730 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:19.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:19 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/39866620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:31:19.691 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:31:19.824 INFO:teuthology.orchestra.run.smithi112.stdout: 5 89756 13.37 MB/sec execute 43 sec latency 35.916 ms 2024-04-26T15:31:20.298 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:20 smithi112 ceph-mon[21780]: pgmap v20: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.3 MiB/s wr, 633 op/s 2024-04-26T15:31:20.298 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:20 smithi112 ceph-mon[21780]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:20.298 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:20 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:20.298 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:20 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:20.298 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:20 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1896457373' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:31:20.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:20 smithi164 ceph-mon[24045]: pgmap v20: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.3 MiB/s wr, 633 op/s 2024-04-26T15:31:20.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:20 smithi164 ceph-mon[24045]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:20.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:20 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:20 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:20 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1896457373' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:31:20.824 INFO:teuthology.orchestra.run.smithi112.stdout: 5 90301 13.35 MB/sec execute 44 sec latency 35.774 ms 2024-04-26T15:31:21.315 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:21 smithi112 ceph-mon[21780]: Reconfiguring node-exporter.smithi112 (dependencies changed)... 2024-04-26T15:31:21.315 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:21 smithi112 ceph-mon[21780]: Reconfiguring daemon node-exporter.smithi112 on smithi112 2024-04-26T15:31:21.498 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:21 smithi164 ceph-mon[24045]: Reconfiguring node-exporter.smithi112 (dependencies changed)... 2024-04-26T15:31:21.499 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:21 smithi164 ceph-mon[24045]: Reconfiguring daemon node-exporter.smithi112 on smithi112 2024-04-26T15:31:21.824 INFO:teuthology.orchestra.run.smithi112.stdout: 5 90857 13.33 MB/sec execute 45 sec latency 33.268 ms 2024-04-26T15:31:22.323 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:22 smithi112 ceph-mon[21780]: pgmap v21: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.3 MiB/s wr, 633 op/s 2024-04-26T15:31:22.514 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:22 smithi164 ceph-mon[24045]: pgmap v21: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.3 MiB/s wr, 633 op/s 2024-04-26T15:31:22.825 INFO:teuthology.orchestra.run.smithi112.stdout: 5 91522 13.47 MB/sec execute 46 sec latency 19.166 ms 2024-04-26T15:31:23.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:23 smithi164 ceph-mon[24045]: pgmap v22: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 900 op/s 2024-04-26T15:31:23.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:23 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:23.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:23 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:23.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:23 smithi164 ceph-mon[24045]: Reconfiguring prometheus.smithi112 (dependencies changed)... 2024-04-26T15:31:23.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:23 smithi164 ceph-mon[24045]: Reconfiguring daemon prometheus.smithi112 on smithi112 2024-04-26T15:31:23.603 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:23 smithi112 ceph-mon[21780]: pgmap v22: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 900 op/s 2024-04-26T15:31:23.603 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:23 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:23.603 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:23 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:23.603 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:23 smithi112 ceph-mon[21780]: Reconfiguring prometheus.smithi112 (dependencies changed)... 2024-04-26T15:31:23.603 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:23 smithi112 ceph-mon[21780]: Reconfiguring daemon prometheus.smithi112 on smithi112 2024-04-26T15:31:23.824 INFO:teuthology.orchestra.run.smithi112.stdout: 5 92134 13.64 MB/sec execute 47 sec latency 45.920 ms 2024-04-26T15:31:24.825 INFO:teuthology.orchestra.run.smithi112.stdout: 5 92623 13.69 MB/sec execute 48 sec latency 40.906 ms 2024-04-26T15:31:25.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:25 smithi112 ceph-mon[21780]: pgmap v23: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.7 MiB/s wr, 557 op/s 2024-04-26T15:31:25.825 INFO:teuthology.orchestra.run.smithi112.stdout: 5 92990 13.63 MB/sec execute 49 sec latency 51.726 ms 2024-04-26T15:31:25.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:25 smithi164 ceph-mon[24045]: pgmap v23: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.7 MiB/s wr, 557 op/s 2024-04-26T15:31:26.825 INFO:teuthology.orchestra.run.smithi112.stdout: 5 93369 13.43 MB/sec execute 50 sec latency 71.185 ms 2024-04-26T15:31:27.825 INFO:teuthology.orchestra.run.smithi112.stdout: 5 93794 13.32 MB/sec execute 51 sec latency 40.135 ms 2024-04-26T15:31:27.941 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:27 smithi112 ceph-mon[21780]: pgmap v24: 97 pgs: 97 active+clean; 340 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 884 op/s 2024-04-26T15:31:28.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:27 smithi164 ceph-mon[24045]: pgmap v24: 97 pgs: 97 active+clean; 340 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 884 op/s 2024-04-26T15:31:28.825 INFO:teuthology.orchestra.run.smithi112.stdout: 5 94263 13.35 MB/sec execute 52 sec latency 58.811 ms 2024-04-26T15:31:29.828 INFO:teuthology.orchestra.run.smithi112.stdout: 5 94908 13.43 MB/sec execute 53 sec latency 28.174 ms 2024-04-26T15:31:29.969 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:29 smithi112 ceph-mon[21780]: pgmap v25: 97 pgs: 97 active+clean; 340 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 593 op/s 2024-04-26T15:31:29.969 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:29 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:31:30.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:29 smithi164 ceph-mon[24045]: pgmap v25: 97 pgs: 97 active+clean; 340 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 593 op/s 2024-04-26T15:31:30.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:29 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:31:30.828 INFO:teuthology.orchestra.run.smithi112.stdout: 5 95439 13.53 MB/sec execute 54 sec latency 35.567 ms 2024-04-26T15:31:31.828 INFO:teuthology.orchestra.run.smithi112.stdout: 5 95950 13.60 MB/sec execute 55 sec latency 29.063 ms 2024-04-26T15:31:32.000 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:31 smithi112 ceph-mon[21780]: pgmap v26: 97 pgs: 97 active+clean; 340 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 593 op/s 2024-04-26T15:31:32.000 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:31 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:32.000 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:31 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:32.061 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:31 smithi164 ceph-mon[24045]: pgmap v26: 97 pgs: 97 active+clean; 340 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 593 op/s 2024-04-26T15:31:32.061 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:31 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:32.061 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:31 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:32.828 INFO:teuthology.orchestra.run.smithi112.stdout: 5 96363 13.54 MB/sec execute 56 sec latency 38.298 ms 2024-04-26T15:31:33.025 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:32 smithi112 ceph-mon[21780]: Reconfiguring node-exporter.smithi164 (dependencies changed)... 2024-04-26T15:31:33.026 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:32 smithi112 ceph-mon[21780]: Reconfiguring daemon node-exporter.smithi164 on smithi164 2024-04-26T15:31:33.029 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:32 smithi164 ceph-mon[24045]: Reconfiguring node-exporter.smithi164 (dependencies changed)... 2024-04-26T15:31:33.030 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:32 smithi164 ceph-mon[24045]: Reconfiguring daemon node-exporter.smithi164 on smithi164 2024-04-26T15:31:33.828 INFO:teuthology.orchestra.run.smithi112.stdout: 5 96805 13.43 MB/sec execute 57 sec latency 36.274 ms 2024-04-26T15:31:34.046 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:33 smithi112 ceph-mon[21780]: pgmap v27: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.1 MiB/s wr, 878 op/s 2024-04-26T15:31:34.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:33 smithi164 ceph-mon[24045]: pgmap v27: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.1 MiB/s wr, 878 op/s 2024-04-26T15:31:34.828 INFO:teuthology.orchestra.run.smithi112.stdout: 5 97164 13.33 MB/sec execute 58 sec latency 48.481 ms 2024-04-26T15:31:35.065 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:35.065 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:35.065 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:31:35.065 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:31:35.065 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi112.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T15:31:35.065 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi112.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T15:31:35.065 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:35.065 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:31:35.065 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:31:35.066 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi112.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T15:31:35.066 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi112.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T15:31:35.066 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:35.066 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:31:35.066 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:31:35.066 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi112.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T15:31:35.066 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi112.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T15:31:35.066 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:35.066 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:31:35.066 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: Upgrade: Need to upgrade myself (mgr.smithi164.tuhwus) 2024-04-26T15:31:35.066 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:34 smithi112 ceph-mon[21780]: Upgrade: Need to upgrade myself (mgr.smithi164.tuhwus) 2024-04-26T15:31:35.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:35.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi112.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi112.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi112.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi112.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi112.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T15:31:35.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi112.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T15:31:35.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:35.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:31:35.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: Upgrade: Need to upgrade myself (mgr.smithi164.tuhwus) 2024-04-26T15:31:35.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:34 smithi164 ceph-mon[24045]: Upgrade: Need to upgrade myself (mgr.smithi164.tuhwus) 2024-04-26T15:31:35.832 INFO:teuthology.orchestra.run.smithi112.stdout: 5 97626 13.33 MB/sec execute 59 sec latency 32.941 ms 2024-04-26T15:31:36.068 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:36 smithi112 ceph-mon[21780]: pgmap v28: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.2 MiB/s wr, 611 op/s 2024-04-26T15:31:36.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:36 smithi164 ceph-mon[24045]: pgmap v28: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.2 MiB/s wr, 611 op/s 2024-04-26T15:31:36.832 INFO:teuthology.orchestra.run.smithi112.stdout: 5 98197 13.30 MB/sec execute 60 sec latency 42.418 ms 2024-04-26T15:31:37.832 INFO:teuthology.orchestra.run.smithi112.stdout: 5 98631 13.30 MB/sec execute 61 sec latency 45.422 ms 2024-04-26T15:31:38.094 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:37 smithi112 ceph-mon[21780]: pgmap v29: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.0 MiB/s wr, 890 op/s 2024-04-26T15:31:38.095 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:37 smithi112 ceph-mon[21780]: Upgrade: Updating mgr.smithi112.llioug 2024-04-26T15:31:38.095 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:37 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:38.095 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:37 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.llioug", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:31:38.095 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:37 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.llioug", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:31:38.095 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:37 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:31:38.095 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:37 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:31:38.095 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:37 smithi112 ceph-mon[21780]: Deploying daemon mgr.smithi112.llioug on smithi112 2024-04-26T15:31:38.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:37 smithi164 ceph-mon[24045]: pgmap v29: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.0 MiB/s wr, 890 op/s 2024-04-26T15:31:38.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:37 smithi164 ceph-mon[24045]: Upgrade: Updating mgr.smithi112.llioug 2024-04-26T15:31:38.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:37 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:38.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:37 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.llioug", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:31:38.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:37 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.llioug", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:31:38.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:37 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:31:38.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:37 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:31:38.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:37 smithi164 ceph-mon[24045]: Deploying daemon mgr.smithi112.llioug on smithi112 2024-04-26T15:31:38.832 INFO:teuthology.orchestra.run.smithi112.stdout: 5 99096 13.36 MB/sec execute 62 sec latency 56.316 ms 2024-04-26T15:31:39.832 INFO:teuthology.orchestra.run.smithi112.stdout: 5 99494 13.39 MB/sec execute 63 sec latency 52.538 ms 2024-04-26T15:31:40.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:39 smithi112 ceph-mon[21780]: pgmap v30: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.5 MiB/s wr, 563 op/s 2024-04-26T15:31:40.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:39 smithi164 ceph-mon[24045]: pgmap v30: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.5 MiB/s wr, 563 op/s 2024-04-26T15:31:40.832 INFO:teuthology.orchestra.run.smithi112.stdout: 5 99776 13.28 MB/sec execute 64 sec latency 54.520 ms 2024-04-26T15:31:41.832 INFO:teuthology.orchestra.run.smithi112.stdout: 5 100117 13.22 MB/sec execute 65 sec latency 38.836 ms 2024-04-26T15:31:41.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:41 smithi112 ceph-mon[21780]: pgmap v31: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.5 MiB/s wr, 563 op/s 2024-04-26T15:31:42.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:41 smithi164 ceph-mon[24045]: pgmap v31: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.5 MiB/s wr, 563 op/s 2024-04-26T15:31:42.833 INFO:teuthology.orchestra.run.smithi112.stdout: 5 100456 13.07 MB/sec execute 66 sec latency 57.979 ms 2024-04-26T15:31:43.832 INFO:teuthology.orchestra.run.smithi112.stdout: 5 100869 13.00 MB/sec execute 67 sec latency 46.079 ms 2024-04-26T15:31:43.995 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:43 smithi112 ceph-mon[21780]: pgmap v32: 97 pgs: 97 active+clean; 339 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 820 op/s 2024-04-26T15:31:43.996 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:43 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:43.996 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:43 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:43.996 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:43 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:31:44.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:43 smithi164 ceph-mon[24045]: pgmap v32: 97 pgs: 97 active+clean; 339 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 820 op/s 2024-04-26T15:31:44.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:43 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:44.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:43 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:44.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:43 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:31:44.832 INFO:teuthology.orchestra.run.smithi112.stdout: 5 101267 12.98 MB/sec execute 68 sec latency 63.564 ms 2024-04-26T15:31:45.553 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:45 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:45.554 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:45 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:45.554 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:45 smithi112 ceph-mon[21780]: pgmap v33: 97 pgs: 97 active+clean; 339 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.7 MiB/s wr, 535 op/s 2024-04-26T15:31:45.554 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:45 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:31:45.832 INFO:teuthology.orchestra.run.smithi112.stdout: 5 101612 12.88 MB/sec execute 69 sec latency 49.093 ms 2024-04-26T15:31:45.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:45 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:45.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:45 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:45.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:45 smithi164 ceph-mon[24045]: pgmap v33: 97 pgs: 97 active+clean; 339 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.7 MiB/s wr, 535 op/s 2024-04-26T15:31:45.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:45 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:31:46.832 INFO:teuthology.orchestra.run.smithi112.stdout: 5 101983 12.91 MB/sec execute 70 sec latency 46.669 ms 2024-04-26T15:31:47.832 INFO:teuthology.orchestra.run.smithi112.stdout: 5 102444 12.92 MB/sec execute 71 sec latency 30.116 ms 2024-04-26T15:31:47.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:47 smithi112 ceph-mon[21780]: pgmap v34: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 765 op/s 2024-04-26T15:31:47.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:47 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:47.845 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:47 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:48.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:47 smithi164 ceph-mon[24045]: pgmap v34: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 765 op/s 2024-04-26T15:31:48.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:47 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:48.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:47 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:48.833 INFO:teuthology.orchestra.run.smithi112.stdout: 5 102860 12.94 MB/sec execute 72 sec latency 35.151 ms 2024-04-26T15:31:49.833 INFO:teuthology.orchestra.run.smithi112.stdout: 5 103232 12.93 MB/sec execute 73 sec latency 35.152 ms 2024-04-26T15:31:49.864 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:49 smithi112 ceph-mon[21780]: pgmap v35: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 486 op/s 2024-04-26T15:31:50.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:49 smithi164 ceph-mon[24045]: pgmap v35: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 486 op/s 2024-04-26T15:31:50.273 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:31:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: 5 103494 12.84 MB/sec execute 74 sec latency 54.143 ms 2024-04-26T15:31:50.875 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:50 smithi112 ceph-mon[21780]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:51.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:50 smithi164 ceph-mon[24045]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:51.441 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:31:51.441 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 starting - - - - 2024-04-26T15:31:51.441 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (9m) 51s ago 9m 6405k - 17.2.0 e1d6a67b021e c91aa1382673 2024-04-26T15:31:51.441 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (9m) 3s ago 9m 6400k - 17.2.0 e1d6a67b021e ed58e2be875d 2024-04-26T15:31:51.441 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 starting - - - - 2024-04-26T15:31:51.441 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (3m) 51s ago 3m 36.2M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:31:51.441 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (3m) 3s ago 3m 13.1M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:31:51.441 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 starting - - - - 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283 running (86s) 3s ago 9m 523M - 19.0.0-3305-ga430f9ed 031ba2747cfd b621bf362ce7 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (11m) 51s ago 11m 48.5M 2048M 17.2.0 e1d6a67b021e 44d7d076db9f 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (9m) 3s ago 9m 40.6M 2048M 17.2.0 e1d6a67b021e 3e800cb663ea 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (3m) 51s ago 3m 247M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (3m) 3s ago 3m 81.3M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 starting - - - - 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (18s) 3s ago 8m 4999k - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (8m) 51s ago 8m 313M 1936M 17.2.0 e1d6a67b021e 72c22bffb3c9 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (7m) 51s ago 7m 248M 1936M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (7m) 51s ago 7m 212M 1936M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (6m) 51s ago 6m 263M 1936M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (6m) 3s ago 6m 358M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (6m) 3s ago 6m 395M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:31:51.442 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (5m) 3s ago 5m 328M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:31:51.443 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (5m) 3s ago 5m 462M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:31:51.443 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 starting - - - - 2024-04-26T15:31:51.833 INFO:teuthology.orchestra.run.smithi112.stdout: 5 103782 12.75 MB/sec execute 75 sec latency 63.976 ms 2024-04-26T15:31:51.892 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:51 smithi112 ceph-mon[21780]: pgmap v36: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 486 op/s 2024-04-26T15:31:51.892 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:51 smithi112 ceph-mon[21780]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:52.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:51 smithi164 ceph-mon[24045]: pgmap v36: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 486 op/s 2024-04-26T15:31:52.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:51 smithi164 ceph-mon[24045]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:52.134 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:31:52.134 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:31:52.134 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:31:52.134 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 1 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 1 2024-04-26T15:31:52.135 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:31:52.136 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:31:52.700 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:31:52.700 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:31:52.700 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:31:52.700 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:31:52.700 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [], 2024-04-26T15:31:52.700 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "", 2024-04-26T15:31:52.700 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading mgr daemons", 2024-04-26T15:31:52.700 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:31:52.700 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:31:52.833 INFO:teuthology.orchestra.run.smithi112.stdout: 5 104037 12.61 MB/sec execute 76 sec latency 67.704 ms 2024-04-26T15:31:52.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:52 smithi112 ceph-mon[21780]: from='client.24507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:52.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:52 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2212716460' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:31:53.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:52 smithi164 ceph-mon[24045]: from='client.24507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:53.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:52 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/2212716460' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:31:53.348 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:31:53.833 INFO:teuthology.orchestra.run.smithi112.stdout: 5 104322 12.54 MB/sec execute 77 sec latency 60.771 ms 2024-04-26T15:31:53.916 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:53 smithi112 ceph-mon[21780]: pgmap v37: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.6 MiB/s wr, 708 op/s 2024-04-26T15:31:53.917 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:53 smithi112 ceph-mon[21780]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:53.917 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:53 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/3352602715' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:31:53.917 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:53 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:53.917 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:53 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:54.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:53 smithi164 ceph-mon[24045]: pgmap v37: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.6 MiB/s wr, 708 op/s 2024-04-26T15:31:54.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:53 smithi164 ceph-mon[24045]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:31:54.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:53 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/3352602715' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:31:54.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:53 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:54.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:53 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:54.833 INFO:teuthology.orchestra.run.smithi112.stdout: 5 104689 12.49 MB/sec execute 78 sec latency 47.220 ms 2024-04-26T15:31:54.975 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:54 smithi112 ceph-mon[21780]: Standby manager daemon smithi112.llioug restarted 2024-04-26T15:31:54.975 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:54 smithi112 ceph-mon[21780]: Standby manager daemon smithi112.llioug started 2024-04-26T15:31:54.975 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:54 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.llioug/crt"}]: dispatch 2024-04-26T15:31:54.975 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:54 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:31:54.975 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:54 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.llioug/key"}]: dispatch 2024-04-26T15:31:54.975 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:54 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:31:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:54 smithi164 ceph-mon[24045]: Standby manager daemon smithi112.llioug restarted 2024-04-26T15:31:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:54 smithi164 ceph-mon[24045]: Standby manager daemon smithi112.llioug started 2024-04-26T15:31:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:54 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.llioug/crt"}]: dispatch 2024-04-26T15:31:55.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:54 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:31:55.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:54 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.llioug/key"}]: dispatch 2024-04-26T15:31:55.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:54 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:31:55.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 105097 12.44 MB/sec execute 79 sec latency 54.970 ms 2024-04-26T15:31:55.985 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:55 smithi112 ceph-mon[21780]: pgmap v38: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 451 op/s 2024-04-26T15:31:55.986 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:55 smithi112 ceph-mon[21780]: mgrmap e28: smithi164.tuhwus(active, since 71s), standbys: smithi112.llioug 2024-04-26T15:31:55.986 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:55 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:55.986 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:55 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:55 smithi164 ceph-mon[24045]: pgmap v38: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 451 op/s 2024-04-26T15:31:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:55 smithi164 ceph-mon[24045]: mgrmap e28: smithi164.tuhwus(active, since 71s), standbys: smithi112.llioug 2024-04-26T15:31:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:55 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:56.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:55 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:31:56.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 105504 12.48 MB/sec execute 80 sec latency 49.281 ms 2024-04-26T15:31:57.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 106133 12.58 MB/sec execute 81 sec latency 29.828 ms 2024-04-26T15:31:58.001 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:57 smithi112 ceph-mon[21780]: pgmap v39: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 653 op/s 2024-04-26T15:31:58.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:57 smithi164 ceph-mon[24045]: pgmap v39: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 653 op/s 2024-04-26T15:31:58.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 106682 12.65 MB/sec execute 82 sec latency 37.246 ms 2024-04-26T15:31:59.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 107266 12.67 MB/sec execute 83 sec latency 46.720 ms 2024-04-26T15:32:00.011 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:59 smithi112 ceph-mon[21780]: pgmap v40: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-04-26T15:32:00.011 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:31:59 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:32:00.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:59 smithi164 ceph-mon[24045]: pgmap v40: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-04-26T15:32:00.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:31:59 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:32:00.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 107946 12.64 MB/sec execute 84 sec latency 40.054 ms 2024-04-26T15:32:01.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 108589 12.68 MB/sec execute 85 sec latency 31.268 ms 2024-04-26T15:32:02.039 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:01 smithi112 ceph-mon[21780]: pgmap v41: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-04-26T15:32:02.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:01 smithi164 ceph-mon[24045]: pgmap v41: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-04-26T15:32:02.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 109128 12.73 MB/sec execute 86 sec latency 32.274 ms 2024-04-26T15:32:03.316 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:03 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:32:03.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:03 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:32:03.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:03 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:32:03.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:03 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:32:03.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:03 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:32:03.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:03 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:32:03.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:03 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:32:03.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:03 smithi112 ceph-mon[21780]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mgr fail", "who": "smithi164.tuhwus"}]: dispatch 2024-04-26T15:32:03.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:03 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mgr fail", "who": "smithi164.tuhwus"}]: dispatch 2024-04-26T15:32:03.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:03 smithi112 ceph-mon[21780]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T15:32:03.408 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:03 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:32:03.408 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:03 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:32:03.408 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:03 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:32:03.409 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:03 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:32:03.409 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:03 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:32:03.409 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:03 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' 2024-04-26T15:32:03.409 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:03 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:32:03.409 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:03 smithi164 ceph-mon[24045]: from='mgr.14676 172.21.15.164:0/3155329306' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mgr fail", "who": "smithi164.tuhwus"}]: dispatch 2024-04-26T15:32:03.409 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:03 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "mgr fail", "who": "smithi164.tuhwus"}]: dispatch 2024-04-26T15:32:03.409 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:03 smithi164 ceph-mon[24045]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T15:32:03.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 109511 12.74 MB/sec execute 87 sec latency 41.267 ms 2024-04-26T15:32:04.352 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd='[{"prefix": "mgr fail", "who": "smithi164.tuhwus"}]': finished 2024-04-26T15:32:04.352 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: mgrmap e29: smithi112.llioug(active, starting, since 0.858422s) 2024-04-26T15:32:04.352 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:32:04.352 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.gbxewm"}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi112.llioug", "id": "smithi112.llioug"}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: Manager daemon smithi112.llioug is now available 2024-04-26T15:32:04.353 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:32:04.354 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:32:04.354 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:32:04.354 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:04 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:32:04.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14676 ' entity='mgr.smithi164.tuhwus' cmd='[{"prefix": "mgr fail", "who": "smithi164.tuhwus"}]': finished 2024-04-26T15:32:04.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: mgrmap e29: smithi112.llioug(active, starting, since 0.858422s) 2024-04-26T15:32:04.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.gbxewm"}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi112.llioug", "id": "smithi112.llioug"}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:32:04.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: Manager daemon smithi112.llioug is now available 2024-04-26T15:32:04.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:32:04.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:32:04.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:32:04.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:04 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:32:04.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 109763 12.68 MB/sec execute 88 sec latency 71.469 ms 2024-04-26T15:32:05.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:05 smithi112 ceph-mon[21780]: [26/Apr/2024:15:32:03] ENGINE Bus STARTING 2024-04-26T15:32:05.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:05 smithi112 ceph-mon[21780]: [26/Apr/2024:15:32:04] ENGINE Serving on http://172.21.15.112:8765 2024-04-26T15:32:05.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:05 smithi112 ceph-mon[21780]: [26/Apr/2024:15:32:04] ENGINE Serving on https://172.21.15.112:7150 2024-04-26T15:32:05.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:05 smithi112 ceph-mon[21780]: [26/Apr/2024:15:32:04] ENGINE Bus STARTED 2024-04-26T15:32:05.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:05 smithi112 ceph-mon[21780]: mgrmap e30: smithi112.llioug(active, since 1.86868s) 2024-04-26T15:32:05.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:05 smithi112 ceph-mon[21780]: pgmap v3: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:32:05.486 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:05 smithi164 ceph-mon[24045]: [26/Apr/2024:15:32:03] ENGINE Bus STARTING 2024-04-26T15:32:05.487 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:05 smithi164 ceph-mon[24045]: [26/Apr/2024:15:32:04] ENGINE Serving on http://172.21.15.112:8765 2024-04-26T15:32:05.487 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:05 smithi164 ceph-mon[24045]: [26/Apr/2024:15:32:04] ENGINE Serving on https://172.21.15.112:7150 2024-04-26T15:32:05.487 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:05 smithi164 ceph-mon[24045]: [26/Apr/2024:15:32:04] ENGINE Bus STARTED 2024-04-26T15:32:05.487 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:05 smithi164 ceph-mon[24045]: mgrmap e30: smithi112.llioug(active, since 1.86868s) 2024-04-26T15:32:05.487 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:05 smithi164 ceph-mon[24045]: pgmap v3: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:32:05.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 110068 12.69 MB/sec execute 89 sec latency 57.208 ms 2024-04-26T15:32:06.330 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:06 smithi164 ceph-mon[24045]: pgmap v4: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:32:06.423 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:06 smithi112 ceph-mon[21780]: pgmap v4: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:32:06.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 110268 12.60 MB/sec execute 90 sec latency 62.856 ms 2024-04-26T15:32:07.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 110494 12.53 MB/sec execute 91 sec latency 80.559 ms 2024-04-26T15:32:07.990 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:07 smithi112 ceph-mon[21780]: mgrmap e31: smithi112.llioug(active, since 4s) 2024-04-26T15:32:07.990 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:07 smithi112 ceph-mon[21780]: pgmap v5: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:32:08.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:07 smithi164 ceph-mon[24045]: mgrmap e31: smithi112.llioug(active, since 4s) 2024-04-26T15:32:08.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:07 smithi164 ceph-mon[24045]: pgmap v5: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:32:08.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 110708 12.45 MB/sec execute 92 sec latency 80.338 ms 2024-04-26T15:32:09.012 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:08 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:09.012 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:08 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:09.227 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:08 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:09.227 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:08 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:09.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 110962 12.35 MB/sec execute 93 sec latency 80.137 ms 2024-04-26T15:32:10.508 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:10 smithi164 ceph-mon[24045]: pgmap v6: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:32:10.508 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:10 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:10.508 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:10 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:10.549 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:10 smithi112 ceph-mon[21780]: pgmap v6: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T15:32:10.549 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:10 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:10.549 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:10 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:10.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 111354 12.30 MB/sec execute 94 sec latency 31.880 ms 2024-04-26T15:32:11.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 111769 12.26 MB/sec execute 95 sec latency 44.864 ms 2024-04-26T15:32:11.855 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:11 smithi112 ceph-mon[21780]: pgmap v7: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 2.9 MiB/s wr, 281 op/s 2024-04-26T15:32:12.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:11 smithi164 ceph-mon[24045]: pgmap v7: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 2.9 MiB/s wr, 281 op/s 2024-04-26T15:32:12.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 112185 12.23 MB/sec execute 96 sec latency 38.284 ms 2024-04-26T15:32:12.877 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:12 smithi112 ceph-mon[21780]: Standby manager daemon smithi164.tuhwus started 2024-04-26T15:32:12.877 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:12 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/2699595708' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/crt"}]: dispatch 2024-04-26T15:32:12.877 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:12 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/2699595708' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:32:12.877 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:12 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/2699595708' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/key"}]: dispatch 2024-04-26T15:32:12.877 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:12 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/2699595708' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:32:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:12 smithi164 ceph-mon[24045]: Standby manager daemon smithi164.tuhwus started 2024-04-26T15:32:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:12 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/2699595708' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/crt"}]: dispatch 2024-04-26T15:32:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:12 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/2699595708' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:32:13.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:12 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/2699595708' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/key"}]: dispatch 2024-04-26T15:32:13.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:12 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/2699595708' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:32:13.834 INFO:teuthology.orchestra.run.smithi112.stdout: 5 112656 12.27 MB/sec execute 97 sec latency 41.378 ms 2024-04-26T15:32:13.910 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:13 smithi112 ceph-mon[21780]: mgrmap e32: smithi112.llioug(active, since 10s), standbys: smithi164.tuhwus 2024-04-26T15:32:13.911 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:13 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi164.tuhwus", "id": "smithi164.tuhwus"}]: dispatch 2024-04-26T15:32:13.911 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:13 smithi112 ceph-mon[21780]: pgmap v8: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.2 MiB/s wr, 218 op/s 2024-04-26T15:32:13.911 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:13 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:13.911 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:13 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:13 smithi164 ceph-mon[24045]: mgrmap e32: smithi112.llioug(active, since 10s), standbys: smithi164.tuhwus 2024-04-26T15:32:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:13 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi164.tuhwus", "id": "smithi164.tuhwus"}]: dispatch 2024-04-26T15:32:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:13 smithi164 ceph-mon[24045]: pgmap v8: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.2 MiB/s wr, 218 op/s 2024-04-26T15:32:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:13 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:13 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:14.840 INFO:teuthology.orchestra.run.smithi112.stdout: 5 113064 12.29 MB/sec execute 98 sec latency 38.206 ms 2024-04-26T15:32:15.840 INFO:teuthology.orchestra.run.smithi112.stdout: 5 113406 12.27 MB/sec execute 99 sec latency 32.985 ms 2024-04-26T15:32:16.472 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:16 smithi112 ceph-mon[21780]: pgmap v9: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.2 MiB/s wr, 445 op/s 2024-04-26T15:32:16.473 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:16 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:16.473 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:16 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:16.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:16 smithi164 ceph-mon[24045]: pgmap v9: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.2 MiB/s wr, 445 op/s 2024-04-26T15:32:16.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:16 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:16.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:16 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:16.840 INFO:teuthology.orchestra.run.smithi112.stdout: 5 113816 12.30 MB/sec execute 100 sec latency 37.453 ms 2024-04-26T15:32:17.845 INFO:teuthology.orchestra.run.smithi112.stdout: 5 114215 12.29 MB/sec execute 101 sec latency 46.134 ms 2024-04-26T15:32:18.497 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:18 smithi112 ceph-mon[21780]: pgmap v10: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.8 MiB/s wr, 405 op/s 2024-04-26T15:32:18.497 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:18 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:32:18.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:18 smithi164 ceph-mon[24045]: pgmap v10: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.8 MiB/s wr, 405 op/s 2024-04-26T15:32:18.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:18 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:32:18.845 INFO:teuthology.orchestra.run.smithi112.stdout: 5 114701 12.24 MB/sec execute 102 sec latency 38.987 ms 2024-04-26T15:32:19.845 INFO:teuthology.orchestra.run.smithi112.stdout: 5 115176 12.20 MB/sec execute 103 sec latency 43.289 ms 2024-04-26T15:32:20.528 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:20 smithi112 ceph-mon[21780]: pgmap v11: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.8 MiB/s wr, 405 op/s 2024-04-26T15:32:20.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:20 smithi164 ceph-mon[24045]: pgmap v11: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.8 MiB/s wr, 405 op/s 2024-04-26T15:32:20.845 INFO:teuthology.orchestra.run.smithi112.stdout: 5 115928 12.29 MB/sec execute 104 sec latency 36.389 ms 2024-04-26T15:32:21.845 INFO:teuthology.orchestra.run.smithi112.stdout: 5 116549 12.34 MB/sec execute 105 sec latency 39.733 ms 2024-04-26T15:32:22.037 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:21 smithi112 ceph-mon[21780]: pgmap v12: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.6 MiB/s wr, 660 op/s 2024-04-26T15:32:22.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:21 smithi164 ceph-mon[24045]: pgmap v12: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.6 MiB/s wr, 660 op/s 2024-04-26T15:32:22.845 INFO:teuthology.orchestra.run.smithi112.stdout: 5 116966 12.36 MB/sec execute 106 sec latency 30.156 ms 2024-04-26T15:32:23.044 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:22 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:23.045 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:22 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:23.045 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:22 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:32:23.045 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:22 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:32:23.045 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:22 smithi112 ceph-mon[21780]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-26T15:32:23.045 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:22 smithi112 ceph-mon[21780]: Updating smithi164:/etc/ceph/ceph.conf 2024-04-26T15:32:23.045 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:22 smithi112 ceph-mon[21780]: Updating smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T15:32:23.045 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:22 smithi112 ceph-mon[21780]: Updating smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T15:32:23.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:22 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:23.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:22 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:23.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:22 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:32:23.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:22 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:32:23.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:22 smithi164 ceph-mon[24045]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-26T15:32:23.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:22 smithi164 ceph-mon[24045]: Updating smithi164:/etc/ceph/ceph.conf 2024-04-26T15:32:23.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:22 smithi164 ceph-mon[24045]: Updating smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T15:32:23.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:22 smithi164 ceph-mon[24045]: Updating smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T15:32:23.841 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:32:23.845 INFO:teuthology.orchestra.run.smithi112.stdout: 5 117274 12.36 MB/sec execute 107 sec latency 44.096 ms 2024-04-26T15:32:24.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:24 smithi112 ceph-mon[21780]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:32:24.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:24 smithi112 ceph-mon[21780]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:32:24.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:24 smithi112 ceph-mon[21780]: Updating smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T15:32:24.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:24 smithi112 ceph-mon[21780]: Updating smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T15:32:24.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:24 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:24.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:24 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:24.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:24 smithi112 ceph-mon[21780]: pgmap v13: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 497 op/s 2024-04-26T15:32:24.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:24 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:24.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:24 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:24.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:24 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:24.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:24 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:24 smithi164 ceph-mon[24045]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:32:24.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:24 smithi164 ceph-mon[24045]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:32:24.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:24 smithi164 ceph-mon[24045]: Updating smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T15:32:24.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:24 smithi164 ceph-mon[24045]: Updating smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T15:32:24.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:24 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:24.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:24 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:24.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:24 smithi164 ceph-mon[24045]: pgmap v13: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 497 op/s 2024-04-26T15:32:24.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:24 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:24.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:24 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:24.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:24 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:24.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:24 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:24.845 INFO:teuthology.orchestra.run.smithi112.stdout: 5 117537 12.30 MB/sec execute 108 sec latency 45.740 ms 2024-04-26T15:32:25.014 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:32:25.014 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (74s) 11s ago 10m 17.7M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:32:25.014 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (10m) 11s ago 10m 6405k - 17.2.0 e1d6a67b021e c91aa1382673 2024-04-26T15:32:25.014 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (9m) 17s ago 9m 6400k - 17.2.0 e1d6a67b021e ed58e2be875d 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (66s) 11s ago 10m 45.0M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (4m) 11s ago 4m 37.7M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (4m) 17s ago 4m 13.3M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (42s) 11s ago 11m 490M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283 running (2m) 17s ago 9m 73.4M - 19.0.0-3305-ga430f9ed 031ba2747cfd b621bf362ce7 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (11m) 11s ago 11m 51.9M 2048M 17.2.0 e1d6a67b021e 44d7d076db9f 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (9m) 17s ago 9m 43.4M 2048M 17.2.0 e1d6a67b021e 3e800cb663ea 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (4m) 11s ago 4m 254M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (3m) 17s ago 3m 83.2M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (63s) 11s ago 10m 8619k - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (51s) 17s ago 9m 9.86M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (8m) 11s ago 8m 431M 1936M 17.2.0 e1d6a67b021e 72c22bffb3c9 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (8m) 11s ago 8m 330M 1936M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (7m) 11s ago 7m 298M 1936M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:32:25.015 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (7m) 11s ago 7m 385M 1936M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:32:25.016 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (7m) 17s ago 7m 382M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:32:25.016 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (6m) 17s ago 6m 421M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:32:25.016 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (6m) 17s ago 6m 349M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:32:25.016 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (5m) 17s ago 5m 485M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:32:25.016 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (54s) 11s ago 9m 53.5M - 2.33.4 514e6a882f6e 4ce827c43009 2024-04-26T15:32:25.362 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:25 smithi112 ceph-mon[21780]: Reconfiguring prometheus.smithi112 (dependencies changed)... 2024-04-26T15:32:25.362 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:25 smithi112 ceph-mon[21780]: Reconfiguring daemon prometheus.smithi112 on smithi112 2024-04-26T15:32:25.362 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:25 smithi112 ceph-mon[21780]: from='client.14792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:32:25.570 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:32:25.570 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:32:25.570 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:32:25.570 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:32:25.570 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:32:25.570 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:32:25.570 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:32:25.570 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:32:25.570 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T15:32:25.570 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:32:25.570 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:32:25.570 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:32:25.571 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:32:25.571 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:32:25.571 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-26T15:32:25.571 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:32:25.571 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:32:25.571 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:32:25.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:25 smithi164 ceph-mon[24045]: Reconfiguring prometheus.smithi112 (dependencies changed)... 2024-04-26T15:32:25.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:25 smithi164 ceph-mon[24045]: Reconfiguring daemon prometheus.smithi112 on smithi112 2024-04-26T15:32:25.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:25 smithi164 ceph-mon[24045]: from='client.14792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:32:25.845 INFO:teuthology.orchestra.run.smithi112.stdout: 5 117833 12.27 MB/sec execute 109 sec latency 59.518 ms 2024-04-26T15:32:26.032 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:32:26.032 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:32:26.033 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:32:26.033 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:32:26.033 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:32:26.033 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr" 2024-04-26T15:32:26.033 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:32:26.033 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "2/23 daemons upgraded", 2024-04-26T15:32:26.033 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "", 2024-04-26T15:32:26.033 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:32:26.033 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:32:26.375 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:26 smithi112 ceph-mon[21780]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:32:26.375 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:26 smithi112 ceph-mon[21780]: from='client.14800 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:32:26.376 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:26 smithi112 ceph-mon[21780]: pgmap v14: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 790 op/s 2024-04-26T15:32:26.376 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:26 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/2026140256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:32:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:26 smithi164 ceph-mon[24045]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:32:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:26 smithi164 ceph-mon[24045]: from='client.14800 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:32:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:26 smithi164 ceph-mon[24045]: pgmap v14: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 790 op/s 2024-04-26T15:32:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:26 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/2026140256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:32:26.611 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:32:26.845 INFO:teuthology.orchestra.run.smithi112.stdout: 5 118229 12.21 MB/sec execute 110 sec latency 96.679 ms 2024-04-26T15:32:27.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:27 smithi112 ceph-mon[21780]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:32:27.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:27 smithi112 ceph-mon[21780]: from='client.? 172.21.15.112:0/1291290628' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:32:27.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:27 smithi164 ceph-mon[24045]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:32:27.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:27 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1291290628' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:32:27.845 INFO:teuthology.orchestra.run.smithi112.stdout: 5 118626 12.15 MB/sec execute 111 sec latency 33.075 ms 2024-04-26T15:32:28.394 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:28 smithi112 ceph-mon[21780]: pgmap v15: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 547 op/s 2024-04-26T15:32:28.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:28 smithi164 ceph-mon[24045]: pgmap v15: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 547 op/s 2024-04-26T15:32:28.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 119108 12.18 MB/sec execute 112 sec latency 42.479 ms 2024-04-26T15:32:29.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 119678 12.22 MB/sec execute 113 sec latency 32.952 ms 2024-04-26T15:32:30.413 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:30 smithi112 ceph-mon[21780]: pgmap v16: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 547 op/s 2024-04-26T15:32:30.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:30 smithi164 ceph-mon[24045]: pgmap v16: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 547 op/s 2024-04-26T15:32:30.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 120220 12.27 MB/sec execute 114 sec latency 25.121 ms 2024-04-26T15:32:31.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 120644 12.28 MB/sec execute 115 sec latency 41.160 ms 2024-04-26T15:32:31.952 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:31 smithi112 ceph-mon[21780]: pgmap v17: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 794 op/s 2024-04-26T15:32:31.953 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:31 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:31.953 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:31 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:31.953 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:31 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:32:31.953 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:31 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi112:9095"}]: dispatch 2024-04-26T15:32:31.953 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:31 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:31.953 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:31 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:32:32.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:31 smithi164 ceph-mon[24045]: pgmap v17: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 794 op/s 2024-04-26T15:32:32.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:31 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:32.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:31 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:32.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:31 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:32:32.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:31 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi112:9095"}]: dispatch 2024-04-26T15:32:32.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:31 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:32.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:31 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:32:32.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 120953 12.25 MB/sec execute 116 sec latency 53.778 ms 2024-04-26T15:32:32.978 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:32 smithi112 ceph-mon[21780]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:32:32.979 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:32 smithi112 ceph-mon[21780]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi112:9095"}]: dispatch 2024-04-26T15:32:33.092 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:32 smithi164 ceph-mon[24045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:32:33.093 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:32 smithi164 ceph-mon[24045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi112:9095"}]: dispatch 2024-04-26T15:32:33.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 121260 12.23 MB/sec execute 117 sec latency 46.216 ms 2024-04-26T15:32:34.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:33 smithi112 ceph-mon[21780]: pgmap v18: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.3 MiB/s wr, 540 op/s 2024-04-26T15:32:34.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:33 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:32:34.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:33 smithi164 ceph-mon[24045]: pgmap v18: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.3 MiB/s wr, 540 op/s 2024-04-26T15:32:34.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:33 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:32:34.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 121742 12.17 MB/sec execute 118 sec latency 40.145 ms 2024-04-26T15:32:34.936 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:34 smithi164 ceph-mon[24045]: Upgrade: Updating mgr.smithi164.tuhwus 2024-04-26T15:32:34.936 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:34 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.tuhwus", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:32:34.936 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:34 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:32:34.936 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:34 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:32:34.936 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:34 smithi164 ceph-mon[24045]: Deploying daemon mgr.smithi164.tuhwus on smithi164 2024-04-26T15:32:35.017 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:34 smithi112 ceph-mon[21780]: Upgrade: Updating mgr.smithi164.tuhwus 2024-04-26T15:32:35.018 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:34 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.tuhwus", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:32:35.018 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:34 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:32:35.018 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:34 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:32:35.018 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:34 smithi112 ceph-mon[21780]: Deploying daemon mgr.smithi164.tuhwus on smithi164 2024-04-26T15:32:35.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 122302 12.19 MB/sec execute 119 sec latency 36.617 ms 2024-04-26T15:32:36.010 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:35 smithi164 ceph-mon[24045]: pgmap v19: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 783 op/s 2024-04-26T15:32:36.029 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:35 smithi112 ceph-mon[21780]: pgmap v19: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 783 op/s 2024-04-26T15:32:36.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 122727 12.16 MB/sec execute 120 sec latency 45.370 ms 2024-04-26T15:32:37.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 123176 12.19 MB/sec execute 121 sec latency 45.249 ms 2024-04-26T15:32:38.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:38 smithi164 ceph-mon[24045]: pgmap v20: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 489 op/s 2024-04-26T15:32:38.456 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:38 smithi112 ceph-mon[21780]: pgmap v20: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 489 op/s 2024-04-26T15:32:38.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 123536 12.16 MB/sec execute 122 sec latency 79.694 ms 2024-04-26T15:32:39.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 123889 12.17 MB/sec execute 123 sec latency 30.559 ms 2024-04-26T15:32:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:39 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:39 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:39 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:32:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:39 smithi164 ceph-mon[24045]: pgmap v21: 97 pgs: 97 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.3 MiB/s wr, 513 op/s 2024-04-26T15:32:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:39 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:39 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:40.241 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:39 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:40.242 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:39 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:40.242 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:39 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:32:40.242 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:39 smithi112 ceph-mon[21780]: pgmap v21: 97 pgs: 97 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.3 MiB/s wr, 513 op/s 2024-04-26T15:32:40.242 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:39 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:40.242 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:39 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:40.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 124297 12.19 MB/sec execute 124 sec latency 40.860 ms 2024-04-26T15:32:41.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 124545 12.14 MB/sec execute 125 sec latency 38.187 ms 2024-04-26T15:32:42.028 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:41 smithi112 ceph-mon[21780]: pgmap v22: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 764 op/s 2024-04-26T15:32:42.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:41 smithi164 ceph-mon[24045]: pgmap v22: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 764 op/s 2024-04-26T15:32:42.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 124863 12.12 MB/sec execute 126 sec latency 46.993 ms 2024-04-26T15:32:43.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 125089 12.05 MB/sec execute 127 sec latency 79.492 ms 2024-04-26T15:32:44.324 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:44 smithi112 ceph-mon[21780]: pgmap v23: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 517 op/s 2024-04-26T15:32:44.405 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:44 smithi164 ceph-mon[24045]: pgmap v23: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 517 op/s 2024-04-26T15:32:44.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 125342 11.98 MB/sec execute 128 sec latency 98.792 ms 2024-04-26T15:32:45.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 125603 11.93 MB/sec execute 129 sec latency 72.731 ms 2024-04-26T15:32:46.117 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:45 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:46.117 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:45 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:46.118 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:45 smithi112 ceph-mon[21780]: pgmap v24: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 712 op/s 2024-04-26T15:32:46.313 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:45 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:46.313 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:45 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:46.313 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:45 smithi164 ceph-mon[24045]: pgmap v24: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 712 op/s 2024-04-26T15:32:46.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 125991 11.93 MB/sec execute 130 sec latency 40.528 ms 2024-04-26T15:32:47.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:47 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:47.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:47 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:47.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:47 smithi164 ceph-mon[24045]: pgmap v25: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 469 op/s 2024-04-26T15:32:47.645 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:47 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:47.645 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:47 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:47.646 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:47 smithi112 ceph-mon[21780]: pgmap v25: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 469 op/s 2024-04-26T15:32:47.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 126401 11.88 MB/sec execute 131 sec latency 62.116 ms 2024-04-26T15:32:48.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 126764 11.90 MB/sec execute 132 sec latency 37.328 ms 2024-04-26T15:32:48.928 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:48 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:48.928 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:48 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:48.929 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:48 smithi112 ceph-mon[21780]: Standby manager daemon smithi164.tuhwus restarted 2024-04-26T15:32:48.929 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:48 smithi112 ceph-mon[21780]: Standby manager daemon smithi164.tuhwus started 2024-04-26T15:32:48.929 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:48 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/3671288628' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/crt"}]: dispatch 2024-04-26T15:32:48.929 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:48 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/3671288628' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:32:48.929 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:48 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/3671288628' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/key"}]: dispatch 2024-04-26T15:32:48.929 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:48 smithi112 ceph-mon[21780]: from='mgr.? 172.21.15.164:0/3671288628' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:32:48.929 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:48 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:32:49.135 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:48 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:49.135 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:48 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:49.135 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:48 smithi164 ceph-mon[24045]: Standby manager daemon smithi164.tuhwus restarted 2024-04-26T15:32:49.135 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:48 smithi164 ceph-mon[24045]: Standby manager daemon smithi164.tuhwus started 2024-04-26T15:32:49.135 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:48 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/3671288628' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/crt"}]: dispatch 2024-04-26T15:32:49.135 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:48 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/3671288628' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:32:49.135 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:48 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/3671288628' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/key"}]: dispatch 2024-04-26T15:32:49.135 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:48 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/3671288628' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:32:49.136 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:48 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:32:49.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 127110 11.89 MB/sec execute 133 sec latency 50.165 ms 2024-04-26T15:32:49.991 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:49 smithi112 ceph-mon[21780]: mgrmap e33: smithi112.llioug(active, since 46s), standbys: smithi164.tuhwus 2024-04-26T15:32:49.991 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:49 smithi112 ceph-mon[21780]: pgmap v26: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 484 op/s 2024-04-26T15:32:50.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:49 smithi164 ceph-mon[24045]: mgrmap e33: smithi112.llioug(active, since 46s), standbys: smithi164.tuhwus 2024-04-26T15:32:50.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:49 smithi164 ceph-mon[24045]: pgmap v26: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 484 op/s 2024-04-26T15:32:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 127529 11.90 MB/sec execute 134 sec latency 50.087 ms 2024-04-26T15:32:51.546 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:51.546 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:51.546 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:32:51.546 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:32:51.546 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:51.546 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:51.546 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:32:51.547 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:32:51.547 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: Upgrade: Setting container_image for all mgr 2024-04-26T15:32:51.547 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:51.547 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi112.llioug"}]: dispatch 2024-04-26T15:32:51.547 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi112.llioug"}]': finished 2024-04-26T15:32:51.547 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi164.tuhwus"}]: dispatch 2024-04-26T15:32:51.547 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi164.tuhwus"}]': finished 2024-04-26T15:32:51.547 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T15:32:51.547 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:51 smithi112 ceph-mon[21780]: pgmap v27: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 649 op/s 2024-04-26T15:32:51.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 127885 11.92 MB/sec execute 135 sec latency 50.755 ms 2024-04-26T15:32:51.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:51.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:51.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:32:51.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:32:51.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:51.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:51.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:32:51.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:32:51.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: Upgrade: Setting container_image for all mgr 2024-04-26T15:32:51.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:51.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi112.llioug"}]: dispatch 2024-04-26T15:32:51.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi112.llioug"}]': finished 2024-04-26T15:32:51.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi164.tuhwus"}]: dispatch 2024-04-26T15:32:51.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi164.tuhwus"}]': finished 2024-04-26T15:32:51.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T15:32:51.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:51 smithi164 ceph-mon[24045]: pgmap v27: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 649 op/s 2024-04-26T15:32:52.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 128203 11.88 MB/sec execute 136 sec latency 45.192 ms 2024-04-26T15:32:53.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 128636 11.87 MB/sec execute 137 sec latency 44.484 ms 2024-04-26T15:32:54.068 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:54 smithi112 ceph-mon[21780]: Upgrade: Updating mon.smithi112 2024-04-26T15:32:54.068 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:54 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:54.069 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:54 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:32:54.069 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:54 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:32:54.069 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:54 smithi112 ceph-mon[21780]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:32:54.069 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:54 smithi112 ceph-mon[21780]: Deploying daemon mon.smithi112 on smithi112 2024-04-26T15:32:54.069 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:54 smithi112 ceph-mon[21780]: pgmap v28: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 399 op/s 2024-04-26T15:32:54.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:54 smithi164 ceph-mon[24045]: Upgrade: Updating mon.smithi112 2024-04-26T15:32:54.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:54 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' 2024-04-26T15:32:54.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:54 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:32:54.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:54 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:32:54.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:54 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:32:54.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:54 smithi164 ceph-mon[24045]: Deploying daemon mon.smithi112 on smithi112 2024-04-26T15:32:54.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:54 smithi164 ceph-mon[24045]: pgmap v28: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 399 op/s 2024-04-26T15:32:54.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 129067 11.84 MB/sec execute 138 sec latency 55.348 ms 2024-04-26T15:32:55.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 129415 11.82 MB/sec execute 139 sec latency 55.442 ms 2024-04-26T15:32:56.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:56 smithi112 systemd[1]: Stopping Ceph mon.smithi112 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:32:56.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:56 smithi112 ceph-mon[21780]: pgmap v29: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.5 MiB/s wr, 634 op/s 2024-04-26T15:32:56.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:56 smithi164 ceph-mon[24045]: pgmap v29: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.5 MiB/s wr, 634 op/s 2024-04-26T15:32:56.597 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:56 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112[21776]: 2024-04-26T15:32:56.511+0000 7f98f4ab3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi112 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:32:56.598 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:56 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112[21776]: 2024-04-26T15:32:56.511+0000 7f98f4ab3700 -1 mon.smithi112@0(leader) e2 *** Got Signal Terminated *** 2024-04-26T15:32:56.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 129779 11.78 MB/sec execute 140 sec latency 55.853 ms 2024-04-26T15:32:56.869 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:56 smithi112 podman[62857]: 2024-04-26 15:32:56.716677613 +0000 UTC m=+0.349469337 container died 44d7d076db9f0c19aa3332b4f7af3752e558546afd63a4481b68b6d57c406aa5 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, version=8, GIT_CLEAN=True, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, release=754, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, architecture=x86_64) 2024-04-26T15:32:57.063 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:32:57.634 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:57 smithi112 podman[62857]: 2024-04-26 15:32:57.382846402 +0000 UTC m=+1.015638127 container cleanup 44d7d076db9f0c19aa3332b4f7af3752e558546afd63a4481b68b6d57c406aa5 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_BRANCH=HEAD, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, distribution-scope=public, io.openshift.expose-services=, version=8, ceph=True) 2024-04-26T15:32:57.634 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:57 smithi112 bash[62857]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112 2024-04-26T15:32:57.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 130111 11.75 MB/sec execute 141 sec latency 41.461 ms 2024-04-26T15:32:57.895 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:57 smithi112 podman[62884]: 2024-04-26 15:32:57.753302123 +0000 UTC m=+1.034824213 container remove 44d7d076db9f0c19aa3332b4f7af3752e558546afd63a4481b68b6d57c406aa5 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, release=754, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, name=centos-stream, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , version=8, io.buildah.version=1.19.8, vendor=Red Hat, Inc.) 2024-04-26T15:32:57.960 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:32:57.960 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (107s) 10s ago 10m 26.1M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:32:57.960 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (10m) 10s ago 10m 6405k - 17.2.0 e1d6a67b021e c91aa1382673 2024-04-26T15:32:57.960 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (10m) 13s ago 10m 6400k - 17.2.0 e1d6a67b021e ed58e2be875d 2024-04-26T15:32:57.960 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (99s) 10s ago 10m 45.5M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:32:57.960 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (4m) 10s ago 4m 36.6M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:32:57.960 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (4m) 13s ago 4m 13.5M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:32:57.960 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (75s) 10s ago 12m 508M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (19s) 13s ago 10m 111M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (12m) 10s ago 12m 51.7M 2048M 17.2.0 e1d6a67b021e 44d7d076db9f 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (10m) 13s ago 10m 40.7M 2048M 17.2.0 e1d6a67b021e 3e800cb663ea 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (4m) 10s ago 4m 251M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (4m) 13s ago 4m 85.4M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (96s) 10s ago 10m 12.0M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (84s) 13s ago 9m 9935k - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (9m) 10s ago 9m 459M 1936M 17.2.0 e1d6a67b021e 72c22bffb3c9 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (8m) 10s ago 8m 404M 1936M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (8m) 10s ago 8m 304M 1936M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (7m) 10s ago 7m 407M 1936M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (7m) 13s ago 7m 433M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:32:57.961 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (7m) 13s ago 7m 463M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:32:57.962 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (6m) 13s ago 6m 406M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:32:57.962 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (6m) 13s ago 6m 542M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:32:57.962 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (27s) 10s ago 9m 51.2M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:32:58.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:58 smithi112 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi112.service: Deactivated successfully. 2024-04-26T15:32:58.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:58 smithi112 systemd[1]: Stopped Ceph mon.smithi112 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:32:58.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:58 smithi112 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi112.service: Consumed 15.464s CPU time. 2024-04-26T15:32:58.409 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:58 smithi112 systemd[1]: Starting Ceph mon.smithi112 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:32:58.535 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:32:58.536 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-26T15:32:58.536 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:32:58.536 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:32:58.536 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:32:58.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:58 smithi112 podman[63095]: 2024-04-26 15:32:58.514620225 +0000 UTC m=+0.025257452 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:32:58.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 130423 11.76 MB/sec execute 142 sec latency 45.198 ms 2024-04-26T15:32:58.916 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:58 smithi112 podman[63095]: 2024-04-26 15:32:58.797073316 +0000 UTC m=+0.307710535 container create d00aea38322941450000dd692173de636714dd99a46a29eb2783feab7cb516c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T15:32:59.023 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:32:59.023 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:32:59.023 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:32:59.024 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:32:59.024 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:32:59.024 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr" 2024-04-26T15:32:59.024 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:32:59.024 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "2/23 daemons upgraded", 2024-04-26T15:32:59.024 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading mon daemons", 2024-04-26T15:32:59.024 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:32:59.024 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:32:59.426 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 podman[63095]: 2024-04-26 15:32:59.197598154 +0000 UTC m=+0.708235375 container init d00aea38322941450000dd692173de636714dd99a46a29eb2783feab7cb516c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424) 2024-04-26T15:32:59.426 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 podman[63095]: 2024-04-26 15:32:59.206501228 +0000 UTC m=+0.717138454 container start d00aea38322941450000dd692173de636714dd99a46a29eb2783feab7cb516c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-26T15:32:59.426 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev), process ceph-mon, pid 2 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: pidfile_write: ignore empty --pid-file 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: load: jerasure load: lrc 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: RocksDB version: 7.9.2 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Git sha 0 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Compile date 2024-04-26 03:22:31 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: DB SUMMARY 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: DB Session ID: 8QZINLM2ACQ1XGBKLEDG 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: CURRENT file: CURRENT 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: MANIFEST file: MANIFEST-000015 size: 979 Bytes 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi112/store.db dir, Total Num: 1, files: 000030.sst 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi112/store.db: 000028.log size: 1543911 ; 2024-04-26T15:32:59.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.error_if_exists: 0 2024-04-26T15:32:59.428 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.create_if_missing: 0 2024-04-26T15:32:59.428 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.paranoid_checks: 1 2024-04-26T15:32:59.428 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T15:32:59.428 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T15:32:59.428 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T15:32:59.428 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.env: 0x55827400acc0 2024-04-26T15:32:59.428 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T15:32:59.428 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.info_log: 0x558275fc5c60 2024-04-26T15:32:59.428 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T15:32:59.429 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.statistics: (nil) 2024-04-26T15:32:59.429 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.use_fsync: 0 2024-04-26T15:32:59.429 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_log_file_size: 0 2024-04-26T15:32:59.429 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T15:32:59.429 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T15:32:59.429 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T15:32:59.429 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T15:32:59.429 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.allow_fallocate: 1 2024-04-26T15:32:59.429 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T15:32:59.429 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T15:32:59.429 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.use_direct_reads: 0 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.db_log_dir: 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.wal_dir: 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.write_buffer_manager: 0x558275f970e0 2024-04-26T15:32:59.430 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.unordered_write: 0 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.row_cache: None 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.wal_filter: None 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T15:32:59.431 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.two_write_queues: 0 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.wal_compression: 0 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.atomic_flush: 0 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.log_readahead_size: 0 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T15:32:59.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_background_jobs: 2 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_background_compactions: -1 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_subcompactions: 1 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_open_files: -1 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T15:32:59.433 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_background_flushes: -1 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Compression algorithms supported: 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: kZSTD supported: 0 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: kXpressCompression supported: 0 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: kZlibCompression supported: 1 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: kSnappyCompression supported: 1 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: kLZ4Compression supported: 1 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: kBZip2Compression supported: 0 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T15:32:59.434 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi112/store.db/MANIFEST-000015 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.merge_operator: 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_filter: None 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558275fc5e20) 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: cache_index_and_filter_blocks: 1 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: pin_top_level_index_and_filter: 1 2024-04-26T15:32:59.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: index_type: 0 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: data_block_index_type: 0 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: index_shortening: 1 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: checksum: 4 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: no_block_cache: 0 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_cache: 0x558275fc91f0 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_cache_name: BinnedLRUCache 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_cache_options: 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: capacity : 536870912 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: num_shard_bits : 4 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: strict_capacity_limit : 0 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: high_pri_pool_ratio: 0.000 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_cache_compressed: (nil) 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: persistent_cache: (nil) 2024-04-26T15:32:59.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_size: 4096 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_size_deviation: 10 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_restart_interval: 16 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: index_block_restart_interval: 1 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: metadata_block_size: 4096 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: partition_filters: 0 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: use_delta_encoding: 1 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: filter_policy: bloomfilter 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: whole_key_filtering: 1 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: verify_compression: 0 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: read_amp_bytes_per_bit: 0 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: format_version: 5 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: enable_index_compression: 1 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_align: 0 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: max_auto_readahead_size: 262144 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: prepopulate_block_cache: 0 2024-04-26T15:32:59.437 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: initial_auto_readahead_size: 8192 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compression: NoCompression 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.num_levels: 7 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T15:32:59.438 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T15:32:59.439 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T15:32:59.440 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T15:32:59.440 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T15:32:59.440 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T15:32:59.440 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T15:32:59.440 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T15:32:59.440 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T15:32:59.440 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T15:32:59.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.inplace_update_support: 0 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T15:32:59.442 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.bloom_locality: 0 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.max_successive_merges: 0 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.ttl: 2592000 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.enable_blob_files: false 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.min_blob_size: 0 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T15:32:59.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi112/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11849, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 74d6ffdc-59ef-41f9-a9fa-7b9b609f0dc8 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714145579247149, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714145579253175, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 1377804, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11843, "largest_seqno": 12689, "table_properties": {"data_size": 1373412, "index_size": 2456, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 901, "raw_key_size": 8393, "raw_average_key_size": 23, "raw_value_size": 1365435, "raw_average_value_size": 3846, "num_data_blocks": 113, "num_entries": 355, "num_filter_entries": 355, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714145579, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "74d6ffdc-59ef-41f9-a9fa-7b9b609f0dc8", "db_session_id": "8QZINLM2ACQ1XGBKLEDG", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714145579253323, "job": 1, "event": "recovery_finished"} 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi112/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5582760e4000 2024-04-26T15:32:59.444 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: DB pointer 0x5582760d0000 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** DB Stats ** 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** Compaction Stats [default] ** 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: L0 1/0 1.31 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 416.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T15:32:59.445 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: L6 1/0 6.94 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Sum 2/0 8.26 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 416.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 416.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** Compaction Stats [default] ** 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 416.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T15:32:59.446 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative compaction: 0.00 GB write, 133.06 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval compaction: 0.00 GB write, 133.06 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Block cache BinnedLRUCache@0x558275fc91f0#2 capacity: 512.00 MB usage: 3.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,2.58 KB,0.000491738%) Misc(1,0.00 KB,0%) 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: starting mon.smithi112 rank 0 at public addrs [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] at bind addrs [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi112 fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: mon.smithi112@-1(???) e2 preinit fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: mon.smithi112@-1(???).mds e7 new map 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: mon.smithi112@-1(???).mds e7 print_map 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: e7 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: legacy client fscid: 1 2024-04-26T15:32:59.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Filesystem 'foofs' (1) 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: fs_name foofs 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: epoch 5 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: created 2024-04-26T15:27:59.809798+0000 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: modified 2024-04-26T15:28:05.834717+0000 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: tableserver 0 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: root 0 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: session_timeout 60 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: session_autoclose 300 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: max_file_size 1099511627776 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: max_xattr_size 65536 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: required_client_features {} 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: last_failure 0 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: last_failure_osd_epoch 0 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T15:32:59.448 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: max_mds 1 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: in 0 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: up {0=14486} 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: failed 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: damaged 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: stopped 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: data_pools [3] 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: metadata_pool 2 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: inline_data disabled 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: balancer 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: bal_rank_mask -1 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: standby_count_wanted 1 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: qdb_cluster leader: 0 members: 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: [mds.foofs.smithi112.fqmway{0:14486} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.112:6834/365553452,v1:172.21.15.112:6835/365553452] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T15:32:59.449 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:32:59.450 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:32:59.450 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Standby daemons: 2024-04-26T15:32:59.450 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-26T15:32:59.450 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: [mds.foofs.smithi164.gbxewm{-1:24263} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.164:6832/3623172766,v1:172.21.15.164:6833/3623172766] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T15:32:59.450 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: mon.smithi112@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T15:32:59.450 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: mon.smithi112@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T15:32:59.450 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: mon.smithi112@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T15:32:59.450 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: mon.smithi112@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T15:32:59.450 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: mon.smithi112@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-04-26T15:32:59.450 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 bash[63095]: d00aea38322941450000dd692173de636714dd99a46a29eb2783feab7cb516c5 2024-04-26T15:32:59.450 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 systemd[1]: Started Ceph mon.smithi112 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:32:59.637 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:32:59.846 INFO:teuthology.orchestra.run.smithi112.stdout: 5 130936 11.80 MB/sec execute 143 sec latency 26.766 ms 2024-04-26T15:33:00.186 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:33:00.186 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:33:00.186 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:00.186 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: pgmap v30: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 439 op/s 2024-04-26T15:33:00.186 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: from='client.24565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: from='client.24569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/1862272567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: from='client.24577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: pgmap v31: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 470 op/s 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/10620660' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: mon.smithi112 calling monitor election 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: mon.smithi112 is new leader, mons smithi112,smithi164 in quorum (ranks 0,1) 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: monmap epoch 2 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: last_changed 2024-04-26T15:22:50.791303+0000 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: created 2024-04-26T15:20:36.122604+0000 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: min_mon_release 17 (quincy) 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: election_strategy: 1 2024-04-26T15:33:00.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: 0: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.smithi112 2024-04-26T15:33:00.188 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: 1: [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon.smithi164 2024-04-26T15:33:00.188 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 1 up:standby 2024-04-26T15:33:00.188 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T15:33:00.188 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: mgrmap e33: smithi112.llioug(active, since 57s), standbys: smithi164.tuhwus 2024-04-26T15:33:00.188 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: overall HEALTH_OK 2024-04-26T15:33:00.188 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: mgrmap e34: smithi112.llioug(active, since 57s), standbys: smithi164.tuhwus 2024-04-26T15:33:00.188 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:32:59 smithi112 ceph-mon[63158]: from='mgr.14746 ' entity='' 2024-04-26T15:33:00.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: from='mgr.14746 172.21.15.112:0/2532367373' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: pgmap v30: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 439 op/s 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: from='client.24565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: from='client.24569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/1862272567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: from='client.24577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: pgmap v31: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 470 op/s 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: from='client.? 172.21.15.112:0/10620660' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: mon.smithi112 calling monitor election 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: mon.smithi112 is new leader, mons smithi112,smithi164 in quorum (ranks 0,1) 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: monmap epoch 2 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: last_changed 2024-04-26T15:22:50.791303+0000 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: created 2024-04-26T15:20:36.122604+0000 2024-04-26T15:33:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: min_mon_release 17 (quincy) 2024-04-26T15:33:00.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: election_strategy: 1 2024-04-26T15:33:00.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: 0: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.smithi112 2024-04-26T15:33:00.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: 1: [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon.smithi164 2024-04-26T15:33:00.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 1 up:standby 2024-04-26T15:33:00.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T15:33:00.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: mgrmap e33: smithi112.llioug(active, since 57s), standbys: smithi164.tuhwus 2024-04-26T15:33:00.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: overall HEALTH_OK 2024-04-26T15:33:00.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: mgrmap e34: smithi112.llioug(active, since 57s), standbys: smithi164.tuhwus 2024-04-26T15:33:00.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:32:59 smithi164 ceph-mon[24045]: from='mgr.14746 ' entity='' 2024-04-26T15:33:00.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 131261 11.80 MB/sec execute 144 sec latency 40.850 ms 2024-04-26T15:33:01.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 131528 11.77 MB/sec execute 145 sec latency 45.535 ms 2024-04-26T15:33:01.957 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:01 smithi112 ceph-mon[63158]: mgrmap e35: smithi112.llioug(active, since 58s), standbys: smithi164.tuhwus 2024-04-26T15:33:02.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:01 smithi164 ceph-mon[24045]: mgrmap e35: smithi112.llioug(active, since 58s), standbys: smithi164.tuhwus 2024-04-26T15:33:02.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 131886 11.76 MB/sec execute 146 sec latency 34.807 ms 2024-04-26T15:33:03.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 132213 11.71 MB/sec execute 147 sec latency 58.653 ms 2024-04-26T15:33:04.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 132459 11.65 MB/sec execute 148 sec latency 102.348 ms 2024-04-26T15:33:05.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 132663 11.61 MB/sec execute 149 sec latency 81.246 ms 2024-04-26T15:33:06.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 132963 11.59 MB/sec execute 150 sec latency 60.674 ms 2024-04-26T15:33:07.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 133246 11.55 MB/sec execute 151 sec latency 84.317 ms 2024-04-26T15:33:08.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 133516 11.51 MB/sec execute 152 sec latency 48.643 ms 2024-04-26T15:33:09.134 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:08 smithi164 ceph-mon[24045]: Standby manager daemon smithi164.tuhwus restarted 2024-04-26T15:33:09.134 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:08 smithi164 ceph-mon[24045]: Standby manager daemon smithi164.tuhwus started 2024-04-26T15:33:09.134 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:08 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/1477364868' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/crt"}]: dispatch 2024-04-26T15:33:09.134 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:08 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/1477364868' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:33:09.134 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:08 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/1477364868' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/key"}]: dispatch 2024-04-26T15:33:09.135 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:08 smithi164 ceph-mon[24045]: from='mgr.? 172.21.15.164:0/1477364868' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:33:09.205 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:08 smithi112 ceph-mon[63158]: Standby manager daemon smithi164.tuhwus restarted 2024-04-26T15:33:09.205 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:08 smithi112 ceph-mon[63158]: Standby manager daemon smithi164.tuhwus started 2024-04-26T15:33:09.205 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:08 smithi112 ceph-mon[63158]: from='mgr.? 172.21.15.164:0/1477364868' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/crt"}]: dispatch 2024-04-26T15:33:09.205 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:08 smithi112 ceph-mon[63158]: from='mgr.? 172.21.15.164:0/1477364868' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:33:09.205 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:08 smithi112 ceph-mon[63158]: from='mgr.? 172.21.15.164:0/1477364868' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.tuhwus/key"}]: dispatch 2024-04-26T15:33:09.205 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:08 smithi112 ceph-mon[63158]: from='mgr.? 172.21.15.164:0/1477364868' entity='mgr.smithi164.tuhwus' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:33:09.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 133840 11.52 MB/sec execute 153 sec latency 42.269 ms 2024-04-26T15:33:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: mgrmap e36: smithi112.llioug(active, since 66s), standbys: smithi164.tuhwus 2024-04-26T15:33:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: Active manager daemon smithi112.llioug restarted 2024-04-26T15:33:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: Activating manager daemon smithi112.llioug 2024-04-26T15:33:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T15:33:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: mgrmap e37: smithi112.llioug(active, starting, since 0.0141317s), standbys: smithi164.tuhwus 2024-04-26T15:33:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:33:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:33:09.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.gbxewm"}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi112.llioug", "id": "smithi112.llioug"}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi164.tuhwus", "id": "smithi164.tuhwus"}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:33:09.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: Manager daemon smithi112.llioug is now available 2024-04-26T15:33:09.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:33:09.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:33:09.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:33:09.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:33:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: mgrmap e36: smithi112.llioug(active, since 66s), standbys: smithi164.tuhwus 2024-04-26T15:33:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: Active manager daemon smithi112.llioug restarted 2024-04-26T15:33:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: Activating manager daemon smithi112.llioug 2024-04-26T15:33:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T15:33:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: mgrmap e37: smithi112.llioug(active, starting, since 0.0141317s), standbys: smithi164.tuhwus 2024-04-26T15:33:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:33:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:33:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:33:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.gbxewm"}]: dispatch 2024-04-26T15:33:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi112.llioug", "id": "smithi112.llioug"}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr metadata", "who": "smithi164.tuhwus", "id": "smithi164.tuhwus"}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: Manager daemon smithi112.llioug is now available 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:33:10.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:33:10.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:33:10.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:09 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.llioug/trash_purge_schedule"}]: dispatch 2024-04-26T15:33:10.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 134257 11.54 MB/sec execute 154 sec latency 45.845 ms 2024-04-26T15:33:11.207 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:10 smithi112 ceph-mon[63158]: [26/Apr/2024:15:33:09] ENGINE Bus STARTING 2024-04-26T15:33:11.208 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:10 smithi112 ceph-mon[63158]: [26/Apr/2024:15:33:09] ENGINE Serving on https://172.21.15.112:7150 2024-04-26T15:33:11.208 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:10 smithi112 ceph-mon[63158]: [26/Apr/2024:15:33:09] ENGINE Serving on http://172.21.15.112:8765 2024-04-26T15:33:11.208 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:10 smithi112 ceph-mon[63158]: [26/Apr/2024:15:33:09] ENGINE Bus STARTED 2024-04-26T15:33:11.208 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:10 smithi112 ceph-mon[63158]: mgrmap e38: smithi112.llioug(active, since 1.01887s), standbys: smithi164.tuhwus 2024-04-26T15:33:11.246 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:10 smithi164 ceph-mon[24045]: [26/Apr/2024:15:33:09] ENGINE Bus STARTING 2024-04-26T15:33:11.246 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:10 smithi164 ceph-mon[24045]: [26/Apr/2024:15:33:09] ENGINE Serving on https://172.21.15.112:7150 2024-04-26T15:33:11.246 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:10 smithi164 ceph-mon[24045]: [26/Apr/2024:15:33:09] ENGINE Serving on http://172.21.15.112:8765 2024-04-26T15:33:11.246 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:10 smithi164 ceph-mon[24045]: [26/Apr/2024:15:33:09] ENGINE Bus STARTED 2024-04-26T15:33:11.246 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:10 smithi164 ceph-mon[24045]: mgrmap e38: smithi112.llioug(active, since 1.01887s), standbys: smithi164.tuhwus 2024-04-26T15:33:11.847 INFO:teuthology.orchestra.run.smithi112.stdout: 5 134620 11.53 MB/sec execute 155 sec latency 42.532 ms 2024-04-26T15:33:12.183 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:11 smithi164 ceph-mon[24045]: pgmap v3: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-26T15:33:12.183 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:11 smithi164 ceph-mon[24045]: pgmap v4: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-26T15:33:12.183 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:11 smithi164 ceph-mon[24045]: mgrmap e39: smithi112.llioug(active, since 2s), standbys: smithi164.tuhwus 2024-04-26T15:33:12.227 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:11 smithi112 ceph-mon[63158]: pgmap v3: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-26T15:33:12.228 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:11 smithi112 ceph-mon[63158]: pgmap v4: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-26T15:33:12.228 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:11 smithi112 ceph-mon[63158]: mgrmap e39: smithi112.llioug(active, since 2s), standbys: smithi164.tuhwus 2024-04-26T15:33:12.852 INFO:teuthology.orchestra.run.smithi112.stdout: 5 134979 11.54 MB/sec execute 156 sec latency 32.833 ms 2024-04-26T15:33:13.853 INFO:teuthology.orchestra.run.smithi112.stdout: 5 135269 11.50 MB/sec execute 157 sec latency 43.453 ms 2024-04-26T15:33:14.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:13 smithi112 ceph-mon[63158]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-26T15:33:14.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:13 smithi112 ceph-mon[63158]: mgrmap e40: smithi112.llioug(active, since 4s), standbys: smithi164.tuhwus 2024-04-26T15:33:14.007 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:14.007 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:14.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:13 smithi164 ceph-mon[24045]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-26T15:33:14.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:13 smithi164 ceph-mon[24045]: mgrmap e40: smithi112.llioug(active, since 4s), standbys: smithi164.tuhwus 2024-04-26T15:33:14.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:13 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:14.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:13 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:14.853 INFO:teuthology.orchestra.run.smithi112.stdout: 5 135602 11.48 MB/sec execute 158 sec latency 53.907 ms 2024-04-26T15:33:15.853 INFO:teuthology.orchestra.run.smithi112.stdout: 5 135913 11.44 MB/sec execute 159 sec latency 53.724 ms 2024-04-26T15:33:16.044 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:15 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:16.044 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:15 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:16.044 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:15 smithi112 ceph-mon[63158]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-26T15:33:16.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:15 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:16.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:15 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:16.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:15 smithi164 ceph-mon[24045]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-26T15:33:16.853 INFO:teuthology.orchestra.run.smithi112.stdout: 5 136256 11.43 MB/sec execute 160 sec latency 65.339 ms 2024-04-26T15:33:17.057 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:17 smithi112 ceph-mon[63158]: pgmap v7: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 34 KiB/s rd, 4.0 MiB/s wr, 379 op/s 2024-04-26T15:33:17.193 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:17 smithi164 ceph-mon[24045]: pgmap v7: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 34 KiB/s rd, 4.0 MiB/s wr, 379 op/s 2024-04-26T15:33:17.853 INFO:teuthology.orchestra.run.smithi112.stdout: 5 136689 11.41 MB/sec execute 161 sec latency 61.513 ms 2024-04-26T15:33:18.853 INFO:teuthology.orchestra.run.smithi112.stdout: 5 137136 11.40 MB/sec execute 162 sec latency 41.361 ms 2024-04-26T15:33:19.092 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:19.092 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:19.093 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:19.093 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:19.093 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:19 smithi112 ceph-mon[63158]: pgmap v8: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.1 MiB/s wr, 294 op/s 2024-04-26T15:33:19.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:19 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:19.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:19 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:19.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:19 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:19.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:19 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:19.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:19 smithi164 ceph-mon[24045]: pgmap v8: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.1 MiB/s wr, 294 op/s 2024-04-26T15:33:19.853 INFO:teuthology.orchestra.run.smithi112.stdout: 5 137757 11.49 MB/sec execute 163 sec latency 30.218 ms 2024-04-26T15:33:20.850 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:20.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:20.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:20.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:33:20.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:20 smithi112 ceph-mon[63158]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-26T15:33:20.851 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:20 smithi112 ceph-mon[63158]: Updating smithi164:/etc/ceph/ceph.conf 2024-04-26T15:33:20.853 INFO:teuthology.orchestra.run.smithi112.stdout: 5 138371 11.52 MB/sec execute 164 sec latency 28.426 ms 2024-04-26T15:33:21.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:20 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:21.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:20 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:21.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:20 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:21.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:20 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:33:21.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:20 smithi164 ceph-mon[24045]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-26T15:33:21.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:20 smithi164 ceph-mon[24045]: Updating smithi164:/etc/ceph/ceph.conf 2024-04-26T15:33:21.853 INFO:teuthology.orchestra.run.smithi112.stdout: 5 139036 11.55 MB/sec execute 165 sec latency 28.271 ms 2024-04-26T15:33:21.855 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: Updating smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T15:33:21.855 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: Updating smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T15:33:21.855 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:33:21.855 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:33:21.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: Updating smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T15:33:21.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: Updating smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T15:33:21.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: pgmap v9: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.5 MiB/s wr, 532 op/s 2024-04-26T15:33:21.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:21.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:21.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:21.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:21.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:21.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:21.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:33:21.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:33:21.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:21 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T15:33:22.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: Updating smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T15:33:22.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: Updating smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T15:33:22.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:33:22.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:33:22.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: Updating smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T15:33:22.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: Updating smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T15:33:22.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: pgmap v9: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.5 MiB/s wr, 532 op/s 2024-04-26T15:33:22.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:22.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:22.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:22.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:22.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:22.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:22.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:33:22.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:33:22.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:21 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T15:33:22.853 INFO:teuthology.orchestra.run.smithi112.stdout: 5 139707 11.57 MB/sec execute 166 sec latency 29.862 ms 2024-04-26T15:33:23.127 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:23 smithi112 ceph-mon[63158]: pgmap v10: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.0 MiB/s wr, 486 op/s 2024-04-26T15:33:23.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:23 smithi164 ceph-mon[24045]: pgmap v10: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.0 MiB/s wr, 486 op/s 2024-04-26T15:33:23.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 140452 11.60 MB/sec execute 167 sec latency 51.483 ms 2024-04-26T15:33:24.640 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:24 smithi112 ceph-mon[63158]: Upgrade: Updating mon.smithi164 2024-04-26T15:33:24.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:24.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:33:24.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:33:24.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:24.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:24 smithi112 ceph-mon[63158]: Deploying daemon mon.smithi164 on smithi164 2024-04-26T15:33:24.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:33:24.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:24 smithi164 ceph-mon[24045]: Upgrade: Updating mon.smithi164 2024-04-26T15:33:24.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:24 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:24.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:24 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:33:24.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:24 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:33:24.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:24 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:24.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:24 smithi164 ceph-mon[24045]: Deploying daemon mon.smithi164 on smithi164 2024-04-26T15:33:24.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:24 smithi164 ceph-mon[24045]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:33:24.853 INFO:teuthology.orchestra.run.smithi112.stdout: 5 141231 11.70 MB/sec execute 168 sec latency 26.913 ms 2024-04-26T15:33:25.644 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:25 smithi112 ceph-mon[63158]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.0 MiB/s wr, 486 op/s 2024-04-26T15:33:25.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:25 smithi164 ceph-mon[24045]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.0 MiB/s wr, 486 op/s 2024-04-26T15:33:25.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:25 smithi164 systemd[1]: Stopping Ceph mon.smithi164 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:33:25.853 INFO:teuthology.orchestra.run.smithi112.stdout: 5 141978 11.76 MB/sec execute 169 sec latency 22.367 ms 2024-04-26T15:33:26.322 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:25 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164[24027]: 2024-04-26T15:33:25.990+0000 7f6e3a7b8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi164 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:33:26.322 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:25 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164[24027]: 2024-04-26T15:33:25.990+0000 7f6e3a7b8700 -1 mon.smithi164@1(peon) e2 *** Got Signal Terminated *** 2024-04-26T15:33:26.322 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:26 smithi164 podman[49797]: 2024-04-26 15:33:26.021336422 +0000 UTC m=+0.177644563 container died 3e800cb663eada6b2a3bae50b7a0a3155741a934747a3275686e11017ad3de53 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, distribution-scope=public, vcs-type=git, architecture=x86_64, vendor=Red Hat, Inc., release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, name=centos-stream, io.openshift.expose-services=, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, build-date=2022-05-03T08:36:31.336870) 2024-04-26T15:33:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:26 smithi164 podman[49797]: 2024-04-26 15:33:26.431222076 +0000 UTC m=+0.587530220 container cleanup 3e800cb663eada6b2a3bae50b7a0a3155741a934747a3275686e11017ad3de53 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164, io.openshift.expose-services=, io.buildah.version=1.19.8, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, release=754, io.openshift.tags=base centos centos-stream, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, version=8, architecture=x86_64, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0) 2024-04-26T15:33:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:26 smithi164 bash[49797]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164 2024-04-26T15:33:26.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 142685 11.80 MB/sec execute 170 sec latency 24.682 ms 2024-04-26T15:33:27.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:26 smithi164 podman[49809]: 2024-04-26 15:33:26.683154101 +0000 UTC m=+0.665526570 container remove 3e800cb663eada6b2a3bae50b7a0a3155741a934747a3275686e11017ad3de53 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164, io.buildah.version=1.19.8, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., vcs-type=git, io.openshift.expose-services=, architecture=x86_64, distribution-scope=public, GIT_BRANCH=HEAD, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8) 2024-04-26T15:33:27.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:26 smithi164 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi164.service: Deactivated successfully. 2024-04-26T15:33:27.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:26 smithi164 systemd[1]: Stopped Ceph mon.smithi164 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:33:27.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:26 smithi164 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi164.service: Consumed 6.945s CPU time. 2024-04-26T15:33:27.381 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 systemd[1]: Starting Ceph mon.smithi164 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:33:27.678 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 podman[49933]: 2024-04-26 15:33:27.284774365 +0000 UTC m=+0.024400054 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:33:27.678 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 podman[49933]: 2024-04-26 15:33:27.426133051 +0000 UTC m=+0.165758738 container create 2e8f0c8e56199e0c8a86883d9c22d1f74daca03bf20cf4c0b6639522c14fb07e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2) 2024-04-26T15:33:27.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 143253 11.79 MB/sec execute 171 sec latency 33.546 ms 2024-04-26T15:33:28.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 podman[49933]: 2024-04-26 15:33:27.679474867 +0000 UTC m=+0.419100554 container init 2e8f0c8e56199e0c8a86883d9c22d1f74daca03bf20cf4c0b6639522c14fb07e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:33:28.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 podman[49933]: 2024-04-26 15:33:27.684127716 +0000 UTC m=+0.423753412 container start 2e8f0c8e56199e0c8a86883d9c22d1f74daca03bf20cf4c0b6639522c14fb07e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:33:28.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T15:33:28.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev), process ceph-mon, pid 2 2024-04-26T15:33:28.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: pidfile_write: ignore empty --pid-file 2024-04-26T15:33:28.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: load: jerasure load: lrc 2024-04-26T15:33:28.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: RocksDB version: 7.9.2 2024-04-26T15:33:28.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Git sha 0 2024-04-26T15:33:28.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Compile date 2024-04-26 03:22:31 2024-04-26T15:33:28.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: DB SUMMARY 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: DB Session ID: 85TXJPDXIZHYLH3W8JE2 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: CURRENT file: CURRENT 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: MANIFEST file: MANIFEST-000009 size: 904 Bytes 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi164/store.db dir, Total Num: 1, files: 000024.sst 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi164/store.db: 000022.log size: 5608966 ; 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.error_if_exists: 0 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.create_if_missing: 0 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.paranoid_checks: 1 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T15:33:28.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.env: 0x555a43edacc0 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.info_log: 0x555a44aa3c60 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.statistics: (nil) 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.use_fsync: 0 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_log_file_size: 0 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.allow_fallocate: 1 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.use_direct_reads: 0 2024-04-26T15:33:28.100 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.db_log_dir: 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.wal_dir: 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.write_buffer_manager: 0x555a44a750e0 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T15:33:28.101 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.unordered_write: 0 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.row_cache: None 2024-04-26T15:33:28.102 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.wal_filter: None 2024-04-26T15:33:28.103 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.two_write_queues: 0 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.wal_compression: 0 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.atomic_flush: 0 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.log_readahead_size: 0 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T15:33:28.104 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_background_jobs: 2 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_background_compactions: -1 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_subcompactions: 1 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_open_files: -1 2024-04-26T15:33:28.105 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_background_flushes: -1 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Compression algorithms supported: 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: kZSTD supported: 0 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: kXpressCompression supported: 0 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: kZlibCompression supported: 1 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: kSnappyCompression supported: 1 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: kLZ4Compression supported: 1 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: kBZip2Compression supported: 0 2024-04-26T15:33:28.106 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi164/store.db/MANIFEST-000009 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.merge_operator: 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_filter: None 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555a44aa3e20) 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: cache_index_and_filter_blocks: 1 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: pin_top_level_index_and_filter: 1 2024-04-26T15:33:28.107 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: index_type: 0 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: data_block_index_type: 0 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: index_shortening: 1 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: checksum: 4 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: no_block_cache: 0 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_cache: 0x555a44aa71f0 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_cache_name: BinnedLRUCache 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_cache_options: 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: capacity : 536870912 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: num_shard_bits : 4 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: strict_capacity_limit : 0 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: high_pri_pool_ratio: 0.000 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_cache_compressed: (nil) 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: persistent_cache: (nil) 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_size: 4096 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_size_deviation: 10 2024-04-26T15:33:28.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_restart_interval: 16 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: index_block_restart_interval: 1 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: metadata_block_size: 4096 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: partition_filters: 0 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: use_delta_encoding: 1 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: filter_policy: bloomfilter 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: whole_key_filtering: 1 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: verify_compression: 0 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: read_amp_bytes_per_bit: 0 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: format_version: 5 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: enable_index_compression: 1 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_align: 0 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: max_auto_readahead_size: 262144 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: prepopulate_block_cache: 0 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: initial_auto_readahead_size: 8192 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T15:33:28.109 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compression: NoCompression 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.num_levels: 7 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T15:33:28.110 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T15:33:28.111 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T15:33:28.112 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T15:33:28.113 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.inplace_update_support: 0 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.bloom_locality: 0 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.max_successive_merges: 0 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T15:33:28.114 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.ttl: 2592000 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.enable_blob_files: false 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.min_blob_size: 0 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T15:33:28.115 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi164/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12224, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: c6d00891-22c2-422b-9bcf-9b30f09e7133 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714145607716249, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714145607731137, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 3464971, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12221, "largest_seqno": 13695, "table_properties": {"data_size": 3458321, "index_size": 4133, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 13732, "raw_average_key_size": 23, "raw_value_size": 3445408, "raw_average_value_size": 5899, "num_data_blocks": 189, "num_entries": 584, "num_filter_entries": 584, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714145607, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "c6d00891-22c2-422b-9bcf-9b30f09e7133", "db_session_id": "85TXJPDXIZHYLH3W8JE2", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714145607731255, "job": 1, "event": "recovery_finished"} 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi164/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x555a44bc4000 2024-04-26T15:33:28.116 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: DB pointer 0x555a44bae000 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** DB Stats ** 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** Compaction Stats [default] ** 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T15:33:28.117 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T15:33:28.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: L0 1/0 3.30 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 621.5 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-04-26T15:33:28.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: L6 1/0 6.94 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T15:33:28.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Sum 2/0 10.25 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 621.5 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-04-26T15:33:28.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 621.5 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-04-26T15:33:28.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-04-26T15:33:28.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** Compaction Stats [default] ** 2024-04-26T15:33:28.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T15:33:28.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:33:28.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 621.5 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-04-26T15:33:28.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-04-26T15:33:28.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T15:33:28.118 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative compaction: 0.00 GB write, 126.32 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval compaction: 0.00 GB write, 126.32 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Block cache BinnedLRUCache@0x555a44aa71f0#2 capacity: 512.00 MB usage: 6.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T15:33:28.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: starting mon.smithi164 rank 1 at public addrs [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] at bind addrs [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi164 fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: mon.smithi164@-1(???) e2 preinit fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: mon.smithi164@-1(???).mds e7 new map 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: mon.smithi164@-1(???).mds e7 print_map 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: e7 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: legacy client fscid: 1 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Filesystem 'foofs' (1) 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: fs_name foofs 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: epoch 5 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: created 2024-04-26T15:27:59.809798+0000 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: modified 2024-04-26T15:28:05.834717+0000 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: tableserver 0 2024-04-26T15:33:28.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: root 0 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: session_timeout 60 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: session_autoclose 300 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: max_file_size 1099511627776 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: max_xattr_size 65536 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: required_client_features {} 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: last_failure 0 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: last_failure_osd_epoch 0 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: max_mds 1 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: in 0 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: up {0=14486} 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: failed 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: damaged 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: stopped 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: data_pools [3] 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: metadata_pool 2 2024-04-26T15:33:28.121 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: inline_data disabled 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: balancer 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: bal_rank_mask -1 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: standby_count_wanted 1 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: qdb_cluster leader: 0 members: 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: [mds.foofs.smithi112.fqmway{0:14486} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.112:6834/365553452,v1:172.21.15.112:6835/365553452] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Standby daemons: 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: [mds.foofs.smithi164.gbxewm{-1:24263} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.164:6832/3623172766,v1:172.21.15.164:6833/3623172766] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: mon.smithi164@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: mon.smithi164@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: mon.smithi164@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: mon.smithi164@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T15:33:28.122 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 ceph-mon[49952]: mon.smithi164@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-04-26T15:33:28.123 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 bash[49933]: 2e8f0c8e56199e0c8a86883d9c22d1f74daca03bf20cf4c0b6639522c14fb07e 2024-04-26T15:33:28.123 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:27 smithi164 systemd[1]: Started Ceph mon.smithi164 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:33:28.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 143862 11.80 MB/sec execute 172 sec latency 30.666 ms 2024-04-26T15:33:29.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:33:29.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:33:29.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: mon.smithi164 calling monitor election 2024-04-26T15:33:29.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: mon.smithi112 calling monitor election 2024-04-26T15:33:29.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: mon.smithi112 is new leader, mons smithi112,smithi164 in quorum (ranks 0,1) 2024-04-26T15:33:29.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: monmap epoch 3 2024-04-26T15:33:29.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:33:29.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: last_changed 2024-04-26T15:33:28.331872+0000 2024-04-26T15:33:29.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: created 2024-04-26T15:20:36.122604+0000 2024-04-26T15:33:29.435 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: min_mon_release 19 (squid) 2024-04-26T15:33:29.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: election_strategy: 1 2024-04-26T15:33:29.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: 0: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.smithi112 2024-04-26T15:33:29.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: 1: [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon.smithi164 2024-04-26T15:33:29.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 1 up:standby 2024-04-26T15:33:29.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T15:33:29.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: mgrmap e40: smithi112.llioug(active, since 19s), standbys: smithi164.tuhwus 2024-04-26T15:33:29.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: overall HEALTH_OK 2024-04-26T15:33:29.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:29.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:29.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:33:29.436 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:29 smithi112 ceph-mon[63158]: pgmap v13: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.9 MiB/s wr, 674 op/s 2024-04-26T15:33:29.567 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-26T15:33:29.567 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-04-26T15:33:29.567 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: mon.smithi164 calling monitor election 2024-04-26T15:33:29.567 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: mon.smithi112 calling monitor election 2024-04-26T15:33:29.567 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: mon.smithi112 is new leader, mons smithi112,smithi164 in quorum (ranks 0,1) 2024-04-26T15:33:29.567 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: monmap epoch 3 2024-04-26T15:33:29.567 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: last_changed 2024-04-26T15:33:28.331872+0000 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: created 2024-04-26T15:20:36.122604+0000 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: min_mon_release 19 (squid) 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: election_strategy: 1 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: 0: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.smithi112 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: 1: [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon.smithi164 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 1 up:standby 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: mgrmap e40: smithi112.llioug(active, since 19s), standbys: smithi164.tuhwus 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: overall HEALTH_OK 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:33:29.568 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:29 smithi164 ceph-mon[49952]: pgmap v13: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.9 MiB/s wr, 674 op/s 2024-04-26T15:33:29.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 144367 11.81 MB/sec execute 173 sec latency 36.170 ms 2024-04-26T15:33:30.185 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:33:30.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 144938 11.86 MB/sec execute 174 sec latency 32.612 ms 2024-04-26T15:33:31.218 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:31 smithi112 ceph-mon[63158]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:31.218 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:31 smithi112 ceph-mon[63158]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:31.218 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:31 smithi112 ceph-mon[63158]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-04-26T15:33:31.239 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:33:31.239 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (2m) 13s ago 11m 26.1M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:33:31.239 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (11m) 13s ago 11m 6405k - 17.2.0 e1d6a67b021e c91aa1382673 2024-04-26T15:33:31.239 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (10m) 17s ago 10m 6400k - 17.2.0 e1d6a67b021e ed58e2be875d 2024-04-26T15:33:31.239 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (2m) 13s ago 11m 44.2M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:33:31.239 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (5m) 13s ago 5m 37.3M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:33:31.239 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (5m) 17s ago 5m 13.7M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (108s) 13s ago 12m 492M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (52s) 17s ago 10m 463M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (32s) 13s ago 12m 31.6M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 starting - - - 2048M 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (5m) 13s ago 5m 253M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (5m) 17s ago 5m 87.0M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (2m) 13s ago 11m 10.9M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (117s) 17s ago 10m 17.1M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (10m) 13s ago 10m 486M 1936M 17.2.0 e1d6a67b021e 72c22bffb3c9 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (9m) 13s ago 9m 411M 1936M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (9m) 13s ago 9m 325M 1936M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (8m) 13s ago 8m 435M 1936M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (8m) 17s ago 8m 482M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (7m) 17s ago 7m 514M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:33:31.240 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (7m) 17s ago 7m 426M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:33:31.241 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (6m) 17s ago 6m 579M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:33:31.241 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (60s) 13s ago 10m 58.0M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:33:31.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:31 smithi164 ceph-mon[49952]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:31.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:31 smithi164 ceph-mon[49952]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:31.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:31 smithi164 ceph-mon[49952]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-04-26T15:33:31.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 145445 11.89 MB/sec execute 175 sec latency 22.833 ms 2024-04-26T15:33:31.921 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:33:31.921 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:33:31.921 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:33:31.921 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:33:31.921 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:33:31.921 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:33:31.921 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:33:31.921 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:33:31.921 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T15:33:31.921 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:33:31.921 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:33:31.922 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:33:31.922 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:33:31.922 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:33:31.922 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-26T15:33:31.922 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 4 2024-04-26T15:33:31.922 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:33:31.922 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:33:32.141 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:32 smithi112 ceph-mon[63158]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:32.141 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:32 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/2571311536' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:33:32.328 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:32 smithi164 ceph-mon[49952]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:32.328 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:32 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/2571311536' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:33:32.404 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:33:32.404 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:33:32.404 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:33:32.404 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:33:32.404 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:33:32.404 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr" 2024-04-26T15:33:32.404 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:33:32.404 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "3/23 daemons upgraded", 2024-04-26T15:33:32.404 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading mon daemons", 2024-04-26T15:33:32.404 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:33:32.404 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:33:32.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 145735 11.87 MB/sec execute 176 sec latency 41.872 ms 2024-04-26T15:33:32.940 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:33:33.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 146039 11.83 MB/sec execute 177 sec latency 46.847 ms 2024-04-26T15:33:33.920 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:33 smithi112 ceph-mon[63158]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:33.921 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:33 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/395909714' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:33:33.921 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:33 smithi112 ceph-mon[63158]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 762 op/s 2024-04-26T15:33:33.921 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:33 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:33.921 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:33 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:34.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:33 smithi164 ceph-mon[49952]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:33:34.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:33 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/395909714' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:33:34.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:33 smithi164 ceph-mon[49952]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 762 op/s 2024-04-26T15:33:34.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:33 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:34.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:33 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:34.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 146334 11.81 MB/sec execute 178 sec latency 72.529 ms 2024-04-26T15:33:35.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 146767 11.80 MB/sec execute 179 sec latency 54.725 ms 2024-04-26T15:33:35.959 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:35 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:35.959 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:35 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:35.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:35 smithi112 ceph-mon[63158]: pgmap v16: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 762 op/s 2024-04-26T15:33:36.088 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:35 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:36.089 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:35 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:36.089 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:35 smithi164 ceph-mon[49952]: pgmap v16: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 762 op/s 2024-04-26T15:33:36.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 147376 11.81 MB/sec execute 180 sec latency 26.731 ms 2024-04-26T15:33:37.201 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:37 smithi164 ceph-mon[49952]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 996 op/s 2024-04-26T15:33:37.226 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:37 smithi112 ceph-mon[63158]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 996 op/s 2024-04-26T15:33:37.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 147930 11.82 MB/sec execute 181 sec latency 27.547 ms 2024-04-26T15:33:38.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 148492 11.87 MB/sec execute 182 sec latency 37.805 ms 2024-04-26T15:33:39.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 148918 11.88 MB/sec execute 183 sec latency 33.555 ms 2024-04-26T15:33:40.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:40.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:40.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:40.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:33:40.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:40.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:40.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: Reconfiguring mon.smithi112 (monmap changed)... 2024-04-26T15:33:40.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:33:40.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:33:40.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:40.007 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: Reconfiguring daemon mon.smithi112 on smithi112 2024-04-26T15:33:40.007 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 MiB/s wr, 589 op/s 2024-04-26T15:33:40.007 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:33:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:33:40.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:40.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:40.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: Reconfiguring mon.smithi112 (monmap changed)... 2024-04-26T15:33:40.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:33:40.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:33:40.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:40.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: Reconfiguring daemon mon.smithi112 on smithi112 2024-04-26T15:33:40.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 MiB/s wr, 589 op/s 2024-04-26T15:33:40.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:33:40.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 149256 11.87 MB/sec execute 184 sec latency 35.970 ms 2024-04-26T15:33:41.276 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:41 smithi112 ceph-mon[63158]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.3 MiB/s wr, 895 op/s 2024-04-26T15:33:41.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:41 smithi164 ceph-mon[49952]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.3 MiB/s wr, 895 op/s 2024-04-26T15:33:41.854 INFO:teuthology.orchestra.run.smithi112.stdout: 5 149566 11.84 MB/sec execute 185 sec latency 59.434 ms 2024-04-26T15:33:42.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 149893 11.82 MB/sec execute 186 sec latency 51.696 ms 2024-04-26T15:33:43.316 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:43 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:43.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:43 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:43.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:43 smithi112 ceph-mon[63158]: Reconfiguring mgr.smithi112.llioug (monmap changed)... 2024-04-26T15:33:43.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:43 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.llioug", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:33:43.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:43 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:33:43.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:43 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:43.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:43 smithi112 ceph-mon[63158]: Reconfiguring daemon mgr.smithi112.llioug on smithi112 2024-04-26T15:33:43.317 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:43 smithi112 ceph-mon[63158]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 540 op/s 2024-04-26T15:33:43.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:43 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:43.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:43 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:43.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:43 smithi164 ceph-mon[49952]: Reconfiguring mgr.smithi112.llioug (monmap changed)... 2024-04-26T15:33:43.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:43 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.llioug", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:33:43.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:43 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:33:43.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:43 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:43.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:43 smithi164 ceph-mon[49952]: Reconfiguring daemon mgr.smithi112.llioug on smithi112 2024-04-26T15:33:43.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:43 smithi164 ceph-mon[49952]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 540 op/s 2024-04-26T15:33:43.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 150233 11.79 MB/sec execute 187 sec latency 40.583 ms 2024-04-26T15:33:44.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 150543 11.76 MB/sec execute 188 sec latency 83.372 ms 2024-04-26T15:33:45.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 150986 11.76 MB/sec execute 189 sec latency 38.693 ms 2024-04-26T15:33:45.881 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:45.882 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:45.882 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:45 smithi112 ceph-mon[63158]: Reconfiguring crash.smithi112 (monmap changed)... 2024-04-26T15:33:45.882 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:33:45.882 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:45.882 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:45 smithi112 ceph-mon[63158]: Reconfiguring daemon crash.smithi112 on smithi112 2024-04-26T15:33:45.882 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:45 smithi112 ceph-mon[63158]: pgmap v21: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 540 op/s 2024-04-26T15:33:46.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:46.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:46.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:45 smithi164 ceph-mon[49952]: Reconfiguring crash.smithi112 (monmap changed)... 2024-04-26T15:33:46.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:33:46.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:46.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:45 smithi164 ceph-mon[49952]: Reconfiguring daemon crash.smithi112 on smithi112 2024-04-26T15:33:46.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:45 smithi164 ceph-mon[49952]: pgmap v21: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 540 op/s 2024-04-26T15:33:46.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 151597 11.80 MB/sec execute 190 sec latency 52.204 ms 2024-04-26T15:33:47.155 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:47 smithi112 ceph-mon[63158]: pgmap v22: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 755 op/s 2024-04-26T15:33:47.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:47 smithi164 ceph-mon[49952]: pgmap v22: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 755 op/s 2024-04-26T15:33:47.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 152157 11.84 MB/sec execute 191 sec latency 74.470 ms 2024-04-26T15:33:48.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 152593 11.84 MB/sec execute 192 sec latency 27.418 ms 2024-04-26T15:33:49.451 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:49 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:49.452 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:49 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:49.452 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:49 smithi112 ceph-mon[63158]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T15:33:49.452 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:49 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T15:33:49.452 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:49 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:49.452 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:49 smithi112 ceph-mon[63158]: Reconfiguring daemon osd.0 on smithi112 2024-04-26T15:33:49.452 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:49 smithi112 ceph-mon[63158]: pgmap v23: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.1 MiB/s wr, 521 op/s 2024-04-26T15:33:49.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:49 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:49.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:49 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:49.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:49 smithi164 ceph-mon[49952]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T15:33:49.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:49 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T15:33:49.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:49 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:49.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:49 smithi164 ceph-mon[49952]: Reconfiguring daemon osd.0 on smithi112 2024-04-26T15:33:49.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:49 smithi164 ceph-mon[49952]: pgmap v23: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.1 MiB/s wr, 521 op/s 2024-04-26T15:33:49.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 153044 11.83 MB/sec execute 193 sec latency 35.769 ms 2024-04-26T15:33:50.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 153447 11.83 MB/sec execute 194 sec latency 37.741 ms 2024-04-26T15:33:51.219 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:51 smithi112 ceph-mon[63158]: pgmap v24: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 811 op/s 2024-04-26T15:33:51.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:51 smithi164 ceph-mon[49952]: pgmap v24: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 811 op/s 2024-04-26T15:33:51.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 153843 11.82 MB/sec execute 195 sec latency 30.825 ms 2024-04-26T15:33:52.479 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:52 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:52.480 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:52 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:52.480 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:52 smithi112 ceph-mon[63158]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T15:33:52.480 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:52 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T15:33:52.480 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:52 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:52.480 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:52 smithi112 ceph-mon[63158]: Reconfiguring daemon osd.1 on smithi112 2024-04-26T15:33:52.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:52 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:52.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:52 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:52.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:52 smithi164 ceph-mon[49952]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T15:33:52.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:52 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T15:33:52.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:52 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:52.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:52 smithi164 ceph-mon[49952]: Reconfiguring daemon osd.1 on smithi112 2024-04-26T15:33:52.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 154195 11.78 MB/sec execute 196 sec latency 47.302 ms 2024-04-26T15:33:53.487 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:53 smithi112 ceph-mon[63158]: pgmap v25: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 505 op/s 2024-04-26T15:33:53.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:53 smithi164 ceph-mon[49952]: pgmap v25: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 505 op/s 2024-04-26T15:33:53.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 154771 11.78 MB/sec execute 197 sec latency 38.588 ms 2024-04-26T15:33:54.493 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:33:54.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:33:54.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 155256 11.83 MB/sec execute 198 sec latency 46.604 ms 2024-04-26T15:33:55.767 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:55 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:55.768 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:55 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:55.768 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:55 smithi112 ceph-mon[63158]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T15:33:55.768 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:55 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T15:33:55.768 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:55 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:55.768 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:55 smithi112 ceph-mon[63158]: Reconfiguring daemon osd.2 on smithi112 2024-04-26T15:33:55.768 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:55 smithi112 ceph-mon[63158]: pgmap v26: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 505 op/s 2024-04-26T15:33:55.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 155734 11.84 MB/sec execute 199 sec latency 50.323 ms 2024-04-26T15:33:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:55 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:55 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:55 smithi164 ceph-mon[49952]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T15:33:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:55 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T15:33:56.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:55 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:56.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:55 smithi164 ceph-mon[49952]: Reconfiguring daemon osd.2 on smithi112 2024-04-26T15:33:56.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:55 smithi164 ceph-mon[49952]: pgmap v26: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 505 op/s 2024-04-26T15:33:56.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 156119 11.84 MB/sec execute 200 sec latency 37.538 ms 2024-04-26T15:33:57.283 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:57 smithi112 ceph-mon[63158]: pgmap v27: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 781 op/s 2024-04-26T15:33:57.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:57 smithi164 ceph-mon[49952]: pgmap v27: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 781 op/s 2024-04-26T15:33:57.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 156392 11.82 MB/sec execute 201 sec latency 35.714 ms 2024-04-26T15:33:58.286 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:58.287 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:58.287 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:58 smithi112 ceph-mon[63158]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T15:33:58.287 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T15:33:58.287 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:58.287 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:58 smithi112 ceph-mon[63158]: Reconfiguring daemon osd.3 on smithi112 2024-04-26T15:33:58.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:58.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:33:58.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:58 smithi164 ceph-mon[49952]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T15:33:58.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T15:33:58.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:33:58.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:58 smithi164 ceph-mon[49952]: Reconfiguring daemon osd.3 on smithi112 2024-04-26T15:33:58.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 156834 11.81 MB/sec execute 202 sec latency 41.923 ms 2024-04-26T15:33:59.529 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:33:59 smithi112 ceph-mon[63158]: pgmap v28: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 566 op/s 2024-04-26T15:33:59.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:33:59 smithi164 ceph-mon[49952]: pgmap v28: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 566 op/s 2024-04-26T15:33:59.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 157391 11.83 MB/sec execute 203 sec latency 40.158 ms 2024-04-26T15:34:00.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 157933 11.82 MB/sec execute 204 sec latency 72.742 ms 2024-04-26T15:34:01.306 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:01 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:01.306 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:01 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:01.306 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:01 smithi112 ceph-mon[63158]: Reconfiguring mds.foofs.smithi112.fqmway (monmap changed)... 2024-04-26T15:34:01.306 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:01 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.fqmway", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T15:34:01.306 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:01 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:01.306 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:01 smithi112 ceph-mon[63158]: Reconfiguring daemon mds.foofs.smithi112.fqmway on smithi112 2024-04-26T15:34:01.306 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:01 smithi112 ceph-mon[63158]: pgmap v29: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 826 op/s 2024-04-26T15:34:01.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:01 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:01.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:01 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:01.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:01 smithi164 ceph-mon[49952]: Reconfiguring mds.foofs.smithi112.fqmway (monmap changed)... 2024-04-26T15:34:01.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:01 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.fqmway", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T15:34:01.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:01 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:01.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:01 smithi164 ceph-mon[49952]: Reconfiguring daemon mds.foofs.smithi112.fqmway on smithi112 2024-04-26T15:34:01.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:01 smithi164 ceph-mon[49952]: pgmap v29: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 826 op/s 2024-04-26T15:34:01.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 158452 11.81 MB/sec execute 205 sec latency 38.724 ms 2024-04-26T15:34:02.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 159040 11.87 MB/sec execute 206 sec latency 39.531 ms 2024-04-26T15:34:03.078 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:03 smithi112 ceph-mon[63158]: pgmap v30: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 536 op/s 2024-04-26T15:34:03.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:03 smithi164 ceph-mon[49952]: pgmap v30: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 536 op/s 2024-04-26T15:34:03.502 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:34:03.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 159507 11.88 MB/sec execute 207 sec latency 29.297 ms 2024-04-26T15:34:04.340 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:04 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:04.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:04 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:04.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:04 smithi112 ceph-mon[63158]: Reconfiguring crash.smithi164 (monmap changed)... 2024-04-26T15:34:04.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:04 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:34:04.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:04 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:04.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:04 smithi112 ceph-mon[63158]: Reconfiguring daemon crash.smithi164 on smithi164 2024-04-26T15:34:04.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:04 smithi112 ceph-mon[63158]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:04.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:04 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:04.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:04 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:04.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:04 smithi164 ceph-mon[49952]: Reconfiguring crash.smithi164 (monmap changed)... 2024-04-26T15:34:04.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:04 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:34:04.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:04 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:04.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:04 smithi164 ceph-mon[49952]: Reconfiguring daemon crash.smithi164 on smithi164 2024-04-26T15:34:04.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:04 smithi164 ceph-mon[49952]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:04.522 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:34:04.522 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (2m) 46s ago 12m 26.1M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:34:04.522 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (12m) 46s ago 12m 6405k - 17.2.0 e1d6a67b021e c91aa1382673 2024-04-26T15:34:04.522 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (11m) 31s ago 11m 6400k - 17.2.0 e1d6a67b021e ed58e2be875d 2024-04-26T15:34:04.522 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (2m) 46s ago 11m 44.2M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:34:04.522 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (6m) 46s ago 5m 37.3M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:34:04.522 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (5m) 31s ago 5m 13.8M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (2m) 46s ago 13m 492M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (86s) 31s ago 11m 463M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (65s) 46s ago 13m 31.6M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (37s) 31s ago 11m 22.4M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (5m) 46s ago 5m 253M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (5m) 31s ago 5m 87.6M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (2m) 46s ago 11m 10.9M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (2m) 31s ago 11m 18.5M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (10m) 46s ago 10m 486M 1936M 17.2.0 e1d6a67b021e 72c22bffb3c9 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (10m) 46s ago 10m 411M 1936M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (9m) 46s ago 9m 325M 1936M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (9m) 46s ago 9m 435M 1936M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:34:04.523 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (8m) 31s ago 8m 489M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:34:04.524 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (8m) 31s ago 8m 554M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:34:04.524 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (7m) 31s ago 7m 463M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:34:04.524 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (7m) 31s ago 7m 353M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:34:04.524 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (93s) 46s ago 10m 58.0M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:34:04.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 159822 11.86 MB/sec execute 208 sec latency 38.571 ms 2024-04-26T15:34:05.149 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:34:05.149 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:34:05.149 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:34:05.149 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 4 2024-04-26T15:34:05.150 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:34:05.151 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:34:05.692 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:34:05.693 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:34:05.693 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:34:05.693 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:34:05.693 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:34:05.693 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:34:05.693 INFO:teuthology.orchestra.run.smithi112.stdout: "mon" 2024-04-26T15:34:05.693 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:34:05.693 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "4/23 daemons upgraded", 2024-04-26T15:34:05.693 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading mon daemons", 2024-04-26T15:34:05.693 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:34:05.693 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:34:05.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 160185 11.84 MB/sec execute 209 sec latency 32.814 ms 2024-04-26T15:34:06.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:05 smithi112 ceph-mon[63158]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:06.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:05 smithi112 ceph-mon[63158]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:06.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:06.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:06.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:05 smithi112 ceph-mon[63158]: Reconfiguring mgr.smithi164.tuhwus (monmap changed)... 2024-04-26T15:34:06.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.tuhwus", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:34:06.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:34:06.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:06.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:05 smithi112 ceph-mon[63158]: Reconfiguring daemon mgr.smithi164.tuhwus on smithi164 2024-04-26T15:34:06.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:05 smithi112 ceph-mon[63158]: pgmap v31: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 536 op/s 2024-04-26T15:34:06.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:05 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/1922979711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:06.329 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:34:06.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:05 smithi164 ceph-mon[49952]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:06.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:05 smithi164 ceph-mon[49952]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:06.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:06.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:06.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:05 smithi164 ceph-mon[49952]: Reconfiguring mgr.smithi164.tuhwus (monmap changed)... 2024-04-26T15:34:06.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.tuhwus", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:34:06.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:34:06.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:06.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:05 smithi164 ceph-mon[49952]: Reconfiguring daemon mgr.smithi164.tuhwus on smithi164 2024-04-26T15:34:06.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:05 smithi164 ceph-mon[49952]: pgmap v31: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 536 op/s 2024-04-26T15:34:06.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:05 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/1922979711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:06.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 160550 11.83 MB/sec execute 210 sec latency 51.444 ms 2024-04-26T15:34:07.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:06 smithi164 ceph-mon[49952]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:07.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:06 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/629908419' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:34:07.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:06 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:07.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:06 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:07.106 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:06 smithi112 ceph-mon[63158]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:07.106 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:06 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/629908419' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:34:07.106 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:06 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:07.106 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:06 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:07.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 160886 11.82 MB/sec execute 211 sec latency 41.774 ms 2024-04-26T15:34:08.110 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:07 smithi112 ceph-mon[63158]: Reconfiguring mon.smithi164 (monmap changed)... 2024-04-26T15:34:08.111 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:07 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:34:08.111 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:07 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:34:08.111 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:07 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:08.111 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:07 smithi112 ceph-mon[63158]: Reconfiguring daemon mon.smithi164 on smithi164 2024-04-26T15:34:08.111 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:07 smithi112 ceph-mon[63158]: pgmap v32: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 827 op/s 2024-04-26T15:34:08.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:07 smithi164 ceph-mon[49952]: Reconfiguring mon.smithi164 (monmap changed)... 2024-04-26T15:34:08.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:07 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:34:08.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:07 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:34:08.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:07 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:08.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:07 smithi164 ceph-mon[49952]: Reconfiguring daemon mon.smithi164 on smithi164 2024-04-26T15:34:08.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:07 smithi164 ceph-mon[49952]: pgmap v32: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 827 op/s 2024-04-26T15:34:08.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 161175 11.79 MB/sec execute 212 sec latency 64.357 ms 2024-04-26T15:34:09.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:09.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:09.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:09 smithi164 ceph-mon[49952]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T15:34:09.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T15:34:09.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:09.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:09 smithi164 ceph-mon[49952]: Reconfiguring daemon osd.4 on smithi164 2024-04-26T15:34:09.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:09 smithi164 ceph-mon[49952]: pgmap v33: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 551 op/s 2024-04-26T15:34:09.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:34:09.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 161586 11.77 MB/sec execute 213 sec latency 65.680 ms 2024-04-26T15:34:09.908 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:09.909 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:09.909 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:09 smithi112 ceph-mon[63158]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T15:34:09.909 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T15:34:09.909 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:09.909 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:09 smithi112 ceph-mon[63158]: Reconfiguring daemon osd.4 on smithi164 2024-04-26T15:34:09.909 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:09 smithi112 ceph-mon[63158]: pgmap v33: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 551 op/s 2024-04-26T15:34:09.909 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:34:10.855 INFO:teuthology.orchestra.run.smithi112.stdout: 5 161867 11.74 MB/sec execute 214 sec latency 43.863 ms 2024-04-26T15:34:11.807 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:11 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:11.808 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:11 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:11.808 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:11 smithi164 ceph-mon[49952]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T15:34:11.808 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:11 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T15:34:11.808 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:11 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:11.808 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:11 smithi164 ceph-mon[49952]: Reconfiguring daemon osd.5 on smithi164 2024-04-26T15:34:11.808 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:11 smithi164 ceph-mon[49952]: pgmap v34: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 763 op/s 2024-04-26T15:34:11.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 162137 11.74 MB/sec execute 215 sec latency 54.759 ms 2024-04-26T15:34:11.972 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:11 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:11.973 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:11 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:11.973 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:11 smithi112 ceph-mon[63158]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T15:34:11.973 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:11 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T15:34:11.973 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:11 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:11.973 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:11 smithi112 ceph-mon[63158]: Reconfiguring daemon osd.5 on smithi164 2024-04-26T15:34:11.973 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:11 smithi112 ceph-mon[63158]: pgmap v34: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 763 op/s 2024-04-26T15:34:12.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 162559 11.76 MB/sec execute 216 sec latency 36.906 ms 2024-04-26T15:34:13.758 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:13.759 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:13.759 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:13 smithi112 ceph-mon[63158]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T15:34:13.759 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T15:34:13.759 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:13.759 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:13 smithi112 ceph-mon[63158]: Reconfiguring daemon osd.6 on smithi164 2024-04-26T15:34:13.759 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:13 smithi112 ceph-mon[63158]: pgmap v35: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-04-26T15:34:13.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 163111 11.78 MB/sec execute 217 sec latency 44.167 ms 2024-04-26T15:34:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:13 smithi164 ceph-mon[49952]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T15:34:14.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T15:34:14.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:14.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:13 smithi164 ceph-mon[49952]: Reconfiguring daemon osd.6 on smithi164 2024-04-26T15:34:14.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:13 smithi164 ceph-mon[49952]: pgmap v35: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-04-26T15:34:14.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 163521 11.78 MB/sec execute 218 sec latency 31.910 ms 2024-04-26T15:34:15.772 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:15 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:15.772 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:15 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:15.773 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:15 smithi112 ceph-mon[63158]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T15:34:15.773 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:15 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T15:34:15.773 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:15 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:15.773 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:15 smithi112 ceph-mon[63158]: Reconfiguring daemon osd.7 on smithi164 2024-04-26T15:34:15.773 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:15 smithi112 ceph-mon[63158]: pgmap v36: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-04-26T15:34:15.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 164119 11.79 MB/sec execute 219 sec latency 33.548 ms 2024-04-26T15:34:16.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:15 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:16.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:15 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:16.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:15 smithi164 ceph-mon[49952]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T15:34:16.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:15 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T15:34:16.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:15 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:16.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:15 smithi164 ceph-mon[49952]: Reconfiguring daemon osd.7 on smithi164 2024-04-26T15:34:16.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:15 smithi164 ceph-mon[49952]: pgmap v36: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-04-26T15:34:16.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 164619 11.79 MB/sec execute 220 sec latency 32.272 ms 2024-04-26T15:34:17.657 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:17 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:17.657 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:17 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:17.657 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:17 smithi164 ceph-mon[49952]: Reconfiguring mds.foofs.smithi164.gbxewm (monmap changed)... 2024-04-26T15:34:17.657 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:17 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.gbxewm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T15:34:17.657 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:17 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:17.658 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:17 smithi164 ceph-mon[49952]: Reconfiguring daemon mds.foofs.smithi164.gbxewm on smithi164 2024-04-26T15:34:17.658 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:17 smithi164 ceph-mon[49952]: pgmap v37: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 745 op/s 2024-04-26T15:34:17.807 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:17 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:17.808 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:17 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:17.808 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:17 smithi112 ceph-mon[63158]: Reconfiguring mds.foofs.smithi164.gbxewm (monmap changed)... 2024-04-26T15:34:17.808 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:17 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.gbxewm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T15:34:17.808 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:17 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:17.808 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:17 smithi112 ceph-mon[63158]: Reconfiguring daemon mds.foofs.smithi164.gbxewm on smithi164 2024-04-26T15:34:17.808 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:17 smithi112 ceph-mon[63158]: pgmap v37: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 745 op/s 2024-04-26T15:34:17.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 165193 11.79 MB/sec execute 221 sec latency 37.326 ms 2024-04-26T15:34:18.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 165679 11.80 MB/sec execute 222 sec latency 30.102 ms 2024-04-26T15:34:19.574 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:19.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:19.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:34:19.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:19.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:19.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:19 smithi112 ceph-mon[63158]: Upgrade: Setting container_image for all mon 2024-04-26T15:34:19.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:19.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi112"}]: dispatch 2024-04-26T15:34:19.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi112"}]': finished 2024-04-26T15:34:19.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi164"}]: dispatch 2024-04-26T15:34:19.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi164"}]': finished 2024-04-26T15:34:19.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:19 smithi112 ceph-mon[63158]: pgmap v38: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 454 op/s 2024-04-26T15:34:19.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:19 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:19.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:19 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:19.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:19 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:34:19.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:19 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:19.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:19 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:19.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:19 smithi164 ceph-mon[49952]: Upgrade: Setting container_image for all mon 2024-04-26T15:34:19.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:19 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:19.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:19 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi112"}]: dispatch 2024-04-26T15:34:19.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:19 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi112"}]': finished 2024-04-26T15:34:19.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:19 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi164"}]: dispatch 2024-04-26T15:34:19.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:19 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi164"}]': finished 2024-04-26T15:34:19.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:19 smithi164 ceph-mon[49952]: pgmap v38: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 454 op/s 2024-04-26T15:34:19.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 166028 11.80 MB/sec execute 223 sec latency 51.708 ms 2024-04-26T15:34:20.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 166398 11.81 MB/sec execute 224 sec latency 42.515 ms 2024-04-26T15:34:21.095 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:21 smithi112 ceph-mon[63158]: pgmap v39: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 755 op/s 2024-04-26T15:34:21.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:21 smithi164 ceph-mon[49952]: pgmap v39: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 755 op/s 2024-04-26T15:34:21.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 166717 11.80 MB/sec execute 225 sec latency 41.235 ms 2024-04-26T15:34:22.369 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:22 smithi112 ceph-mon[63158]: Upgrade: Updating crash.smithi112 (1/2) 2024-04-26T15:34:22.369 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:22.369 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:34:22.369 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:22.369 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:22 smithi112 ceph-mon[63158]: Deploying daemon crash.smithi112 on smithi112 2024-04-26T15:34:22.512 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:22 smithi164 ceph-mon[49952]: Upgrade: Updating crash.smithi112 (1/2) 2024-04-26T15:34:22.512 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:22.512 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:34:22.513 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:22.513 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:22 smithi164 ceph-mon[49952]: Deploying daemon crash.smithi112 on smithi112 2024-04-26T15:34:22.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 166948 11.77 MB/sec execute 226 sec latency 44.787 ms 2024-04-26T15:34:23.410 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:23 smithi112 ceph-mon[63158]: pgmap v40: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.7 MiB/s wr, 542 op/s 2024-04-26T15:34:23.516 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:23 smithi164 ceph-mon[49952]: pgmap v40: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.7 MiB/s wr, 542 op/s 2024-04-26T15:34:23.857 INFO:teuthology.orchestra.run.smithi112.stdout: 5 167254 11.74 MB/sec execute 227 sec latency 41.813 ms 2024-04-26T15:34:24.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:34:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:34:24.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 167553 11.72 MB/sec execute 228 sec latency 47.280 ms 2024-04-26T15:34:25.249 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:25 smithi112 ceph-mon[63158]: pgmap v41: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.2 MiB/s wr, 563 op/s 2024-04-26T15:34:25.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:25 smithi164 ceph-mon[49952]: pgmap v41: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.2 MiB/s wr, 563 op/s 2024-04-26T15:34:25.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 167886 11.70 MB/sec execute 229 sec latency 46.618 ms 2024-04-26T15:34:26.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 168162 11.69 MB/sec execute 230 sec latency 59.639 ms 2024-04-26T15:34:27.185 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:27 smithi112 ceph-mon[63158]: pgmap v42: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 731 op/s 2024-04-26T15:34:27.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:27 smithi164 ceph-mon[49952]: pgmap v42: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 731 op/s 2024-04-26T15:34:27.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 168502 11.68 MB/sec execute 231 sec latency 44.491 ms 2024-04-26T15:34:28.226 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:28.226 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:28.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:28.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:28.857 INFO:teuthology.orchestra.run.smithi112.stdout: 5 168822 11.65 MB/sec execute 232 sec latency 60.031 ms 2024-04-26T15:34:29.245 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:29 smithi112 ceph-mon[63158]: pgmap v43: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.9 MiB/s wr, 489 op/s 2024-04-26T15:34:29.245 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:29 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:29.246 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:29 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:34:29.246 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:29 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:29.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:29 smithi164 ceph-mon[49952]: pgmap v43: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.9 MiB/s wr, 489 op/s 2024-04-26T15:34:29.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:29 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:29.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:29 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-26T15:34:29.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:29 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:29.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 169102 11.62 MB/sec execute 233 sec latency 54.069 ms 2024-04-26T15:34:30.284 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:30 smithi112 ceph-mon[63158]: Upgrade: Updating crash.smithi164 (2/2) 2024-04-26T15:34:30.284 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:30 smithi112 ceph-mon[63158]: Deploying daemon crash.smithi164 on smithi164 2024-04-26T15:34:30.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:30 smithi164 ceph-mon[49952]: Upgrade: Updating crash.smithi164 (2/2) 2024-04-26T15:34:30.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:30 smithi164 ceph-mon[49952]: Deploying daemon crash.smithi164 on smithi164 2024-04-26T15:34:30.856 INFO:teuthology.orchestra.run.smithi112.stdout: 5 169435 11.64 MB/sec execute 234 sec latency 96.443 ms 2024-04-26T15:34:31.296 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:31 smithi112 ceph-mon[63158]: pgmap v44: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 681 op/s 2024-04-26T15:34:31.400 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:31 smithi164 ceph-mon[49952]: pgmap v44: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 681 op/s 2024-04-26T15:34:31.857 INFO:teuthology.orchestra.run.smithi112.stdout: 5 169799 11.64 MB/sec execute 235 sec latency 49.366 ms 2024-04-26T15:34:32.857 INFO:teuthology.orchestra.run.smithi112.stdout: 5 170194 11.64 MB/sec execute 236 sec latency 39.054 ms 2024-04-26T15:34:33.080 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:33 smithi112 ceph-mon[63158]: pgmap v45: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.7 MiB/s wr, 380 op/s 2024-04-26T15:34:33.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:33 smithi164 ceph-mon[49952]: pgmap v45: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.7 MiB/s wr, 380 op/s 2024-04-26T15:34:33.857 INFO:teuthology.orchestra.run.smithi112.stdout: 5 170373 11.61 MB/sec execute 237 sec latency 70.852 ms 2024-04-26T15:34:34.361 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:34.362 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:34.362 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:34:34.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:34.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:34.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:34:34.857 INFO:teuthology.orchestra.run.smithi112.stdout: 5 170520 11.58 MB/sec execute 238 sec latency 141.471 ms 2024-04-26T15:34:35.376 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:35 smithi112 ceph-mon[63158]: pgmap v46: 97 pgs: 97 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 470 op/s 2024-04-26T15:34:35.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:35 smithi164 ceph-mon[49952]: pgmap v46: 97 pgs: 97 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 470 op/s 2024-04-26T15:34:35.857 INFO:teuthology.orchestra.run.smithi112.stdout: 5 170756 11.55 MB/sec execute 239 sec latency 56.405 ms 2024-04-26T15:34:36.803 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:34:36.857 INFO:teuthology.orchestra.run.smithi112.stdout: 5 171006 11.53 MB/sec execute 240 sec latency 67.812 ms 2024-04-26T15:34:37.176 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:37 smithi112 ceph-mon[63158]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:37.177 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:37 smithi112 ceph-mon[63158]: pgmap v47: 97 pgs: 97 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.1 MiB/s wr, 529 op/s 2024-04-26T15:34:37.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:37 smithi164 ceph-mon[49952]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:37.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:37 smithi164 ceph-mon[49952]: pgmap v47: 97 pgs: 97 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.1 MiB/s wr, 529 op/s 2024-04-26T15:34:37.857 INFO:teuthology.orchestra.run.smithi112.stdout: 5 171337 11.51 MB/sec execute 241 sec latency 78.145 ms 2024-04-26T15:34:37.904 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:34:37.904 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (3m) 79s ago 12m 26.1M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:34:37.904 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 starting - - - - 2024-04-26T15:34:37.904 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (5s) 0s ago 11m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:34:37.904 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (3m) 79s ago 12m 44.2M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:34:37.904 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (6m) 79s ago 6m 37.3M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:34:37.904 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (6m) 0s ago 6m 14.0M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:34:37.904 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (2m) 79s ago 13m 492M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:34:37.904 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (119s) 0s ago 11m 465M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:34:37.904 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (99s) 79s ago 13m 31.6M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:34:37.904 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (70s) 0s ago 11m 35.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (6m) 79s ago 6m 253M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (6m) 0s ago 6m 87.6M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (3m) 79s ago 12m 10.9M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (3m) 0s ago 11m 17.6M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (11m) 79s ago 11m 486M 1936M 17.2.0 e1d6a67b021e 72c22bffb3c9 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (10m) 79s ago 10m 411M 1936M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (10m) 79s ago 10m 325M 1936M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (9m) 79s ago 9m 435M 1936M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (9m) 0s ago 9m 551M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (8m) 0s ago 8m 373M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (8m) 0s ago 8m 557M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (8m) 0s ago 8m 448M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:34:37.905 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (2m) 79s ago 11m 58.0M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:34:38.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:38 smithi112 ceph-mon[63158]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:38.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:38 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:38.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:38 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:38.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:38 smithi112 ceph-mon[63158]: from='client.34186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:38.596 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:34:38.596 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:34:38.596 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 4 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:34:38.597 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:34:38.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:38 smithi164 ceph-mon[49952]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:38.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:38 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:38.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:38 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:38.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:38 smithi164 ceph-mon[49952]: from='client.34186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:38.857 INFO:teuthology.orchestra.run.smithi112.stdout: 5 171640 11.51 MB/sec execute 242 sec latency 36.752 ms 2024-04-26T15:34:39.177 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:34:39.178 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:34:39.178 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:34:39.178 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:34:39.178 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:34:39.178 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:34:39.178 INFO:teuthology.orchestra.run.smithi112.stdout: "mon" 2024-04-26T15:34:39.178 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:34:39.178 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "5/23 daemons upgraded", 2024-04-26T15:34:39.178 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading crash daemons", 2024-04-26T15:34:39.178 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:34:39.178 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:34:39.483 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:39 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/1654917021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:39.484 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:39.484 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:39.484 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:39 smithi112 ceph-mon[63158]: pgmap v48: 97 pgs: 97 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 361 op/s 2024-04-26T15:34:39.484 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:34:39.573 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:39 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/1654917021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:39.574 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:39.574 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:39.574 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:39 smithi164 ceph-mon[49952]: pgmap v48: 97 pgs: 97 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 361 op/s 2024-04-26T15:34:39.574 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:34:39.858 INFO:teuthology.orchestra.run.smithi112.stdout: 5 171978 11.48 MB/sec execute 243 sec latency 44.882 ms 2024-04-26T15:34:39.886 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:34:40.749 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:40 smithi112 ceph-mon[63158]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:40.749 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:40 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/1324345650' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:34:40.858 INFO:teuthology.orchestra.run.smithi112.stdout: 5 172357 11.47 MB/sec execute 244 sec latency 37.827 ms 2024-04-26T15:34:41.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:40 smithi164 ceph-mon[49952]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:34:41.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:40 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/1324345650' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:34:41.775 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:41 smithi112 ceph-mon[63158]: pgmap v49: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.2 MiB/s wr, 544 op/s 2024-04-26T15:34:41.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:41 smithi164 ceph-mon[49952]: pgmap v49: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.2 MiB/s wr, 544 op/s 2024-04-26T15:34:41.858 INFO:teuthology.orchestra.run.smithi112.stdout: 5 172677 11.45 MB/sec execute 245 sec latency 35.152 ms 2024-04-26T15:34:42.812 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:42 smithi112 ceph-mon[63158]: mgrmap e41: smithi112.llioug(active, since 92s), standbys: smithi164.tuhwus 2024-04-26T15:34:42.813 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:42 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:42.813 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:42 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:42.813 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:42 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:42.813 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:42 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:42.858 INFO:teuthology.orchestra.run.smithi112.stdout: 5 172962 11.46 MB/sec execute 246 sec latency 38.072 ms 2024-04-26T15:34:43.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:42 smithi164 ceph-mon[49952]: mgrmap e41: smithi112.llioug(active, since 92s), standbys: smithi164.tuhwus 2024-04-26T15:34:43.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:42 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:43.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:42 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:43.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:42 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:43.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:42 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:43.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:43 smithi112 ceph-mon[63158]: pgmap v50: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 352 op/s 2024-04-26T15:34:43.858 INFO:teuthology.orchestra.run.smithi112.stdout: 5 173297 11.46 MB/sec execute 247 sec latency 58.223 ms 2024-04-26T15:34:44.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:43 smithi164 ceph-mon[49952]: pgmap v50: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 352 op/s 2024-04-26T15:34:44.858 INFO:teuthology.orchestra.run.smithi112.stdout: 5 173633 11.45 MB/sec execute 248 sec latency 72.674 ms 2024-04-26T15:34:45.112 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:45.112 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:45.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:45.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:45.858 INFO:teuthology.orchestra.run.smithi112.stdout: 5 173921 11.44 MB/sec execute 249 sec latency 47.053 ms 2024-04-26T15:34:46.143 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:46 smithi112 ceph-mon[63158]: pgmap v51: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 362 op/s 2024-04-26T15:34:46.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:46 smithi164 ceph-mon[49952]: pgmap v51: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 362 op/s 2024-04-26T15:34:46.858 INFO:teuthology.orchestra.run.smithi112.stdout: 5 174189 11.42 MB/sec execute 250 sec latency 48.958 ms 2024-04-26T15:34:47.860 INFO:teuthology.orchestra.run.smithi112.stdout: 5 174537 11.40 MB/sec execute 251 sec latency 46.057 ms 2024-04-26T15:34:48.161 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:48 smithi112 ceph-mon[63158]: pgmap v52: 97 pgs: 97 active+clean; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-04-26T15:34:48.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:48 smithi164 ceph-mon[49952]: pgmap v52: 97 pgs: 97 active+clean; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-04-26T15:34:48.860 INFO:teuthology.orchestra.run.smithi112.stdout: 5 175018 11.40 MB/sec execute 252 sec latency 57.050 ms 2024-04-26T15:34:49.860 INFO:teuthology.orchestra.run.smithi112.stdout: 5 175522 11.41 MB/sec execute 253 sec latency 29.124 ms 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: pgmap v53: 97 pgs: 97 active+clean; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 373 op/s 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:50.194 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi112"}]: dispatch 2024-04-26T15:34:50.195 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi112"}]': finished 2024-04-26T15:34:50.195 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi164"}]: dispatch 2024-04-26T15:34:50.195 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi164"}]': finished 2024-04-26T15:34:50.195 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T15:34:50.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: pgmap v53: 97 pgs: 97 active+clean; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 373 op/s 2024-04-26T15:34:50.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:50.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:50.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi112"}]: dispatch 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi112"}]': finished 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi164"}]: dispatch 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi164"}]': finished 2024-04-26T15:34:50.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T15:34:50.860 INFO:teuthology.orchestra.run.smithi112.stdout: 5 176008 11.40 MB/sec execute 254 sec latency 48.258 ms 2024-04-26T15:34:51.207 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:51 smithi112 ceph-mon[63158]: Upgrade: Setting container_image for all crash 2024-04-26T15:34:51.207 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:51 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T15:34:51.208 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:51 smithi112 ceph-mon[63158]: Upgrade: osd.0 is safe to restart 2024-04-26T15:34:51.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:51 smithi164 ceph-mon[49952]: Upgrade: Setting container_image for all crash 2024-04-26T15:34:51.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:51 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T15:34:51.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:51 smithi164 ceph-mon[49952]: Upgrade: osd.0 is safe to restart 2024-04-26T15:34:51.860 INFO:teuthology.orchestra.run.smithi112.stdout: 5 176437 11.41 MB/sec execute 255 sec latency 51.395 ms 2024-04-26T15:34:52.241 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:52 smithi112 ceph-mon[63158]: pgmap v54: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.8 MiB/s wr, 614 op/s 2024-04-26T15:34:52.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:52 smithi164 ceph-mon[49952]: pgmap v54: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.8 MiB/s wr, 614 op/s 2024-04-26T15:34:52.860 INFO:teuthology.orchestra.run.smithi112.stdout: 5 176859 11.43 MB/sec execute 256 sec latency 46.219 ms 2024-04-26T15:34:53.860 INFO:teuthology.orchestra.run.smithi112.stdout: 5 177357 11.45 MB/sec execute 257 sec latency 42.355 ms 2024-04-26T15:34:54.259 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:54 smithi112 ceph-mon[63158]: pgmap v55: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 431 op/s 2024-04-26T15:34:54.260 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:54.260 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T15:34:54.260 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:54.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:54 smithi164 ceph-mon[49952]: pgmap v55: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 431 op/s 2024-04-26T15:34:54.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:34:54.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T15:34:54.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:34:54.860 INFO:teuthology.orchestra.run.smithi112.stdout: 5 177679 11.43 MB/sec execute 258 sec latency 35.611 ms 2024-04-26T15:34:55.265 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:55 smithi112 ceph-mon[63158]: Upgrade: Updating osd.0 2024-04-26T15:34:55.266 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:55 smithi112 ceph-mon[63158]: Deploying daemon osd.0 on smithi112 2024-04-26T15:34:55.266 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:55 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:34:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:55 smithi164 ceph-mon[49952]: Upgrade: Updating osd.0 2024-04-26T15:34:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:55 smithi164 ceph-mon[49952]: Deploying daemon osd.0 on smithi112 2024-04-26T15:34:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:55 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:34:55.860 INFO:teuthology.orchestra.run.smithi112.stdout: 5 178153 11.42 MB/sec execute 259 sec latency 40.748 ms 2024-04-26T15:34:56.268 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:56 smithi112 ceph-mon[63158]: pgmap v56: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.4 MiB/s wr, 467 op/s 2024-04-26T15:34:56.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:56 smithi164 ceph-mon[49952]: pgmap v56: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.4 MiB/s wr, 467 op/s 2024-04-26T15:34:56.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 178588 11.42 MB/sec execute 260 sec latency 28.924 ms 2024-04-26T15:34:57.289 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:57 smithi112 ceph-mon[63158]: pgmap v57: 97 pgs: 97 active+clean; 353 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.2 MiB/s wr, 684 op/s 2024-04-26T15:34:57.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:57 smithi164 ceph-mon[49952]: pgmap v57: 97 pgs: 97 active+clean; 353 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.2 MiB/s wr, 684 op/s 2024-04-26T15:34:57.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 179020 11.42 MB/sec execute 261 sec latency 39.803 ms 2024-04-26T15:34:58.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 179523 11.41 MB/sec execute 262 sec latency 44.680 ms 2024-04-26T15:34:59.165 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:34:59 smithi112 ceph-mon[63158]: pgmap v58: 97 pgs: 97 active+clean; 353 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 503 op/s 2024-04-26T15:34:59.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:34:59 smithi164 ceph-mon[49952]: pgmap v58: 97 pgs: 97 active+clean; 353 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 503 op/s 2024-04-26T15:34:59.419 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:34:59 smithi112 systemd[1]: Stopping Ceph osd.0 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:34:59.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 179868 11.42 MB/sec execute 263 sec latency 36.933 ms 2024-04-26T15:34:59.925 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:34:59 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0[31967]: 2024-04-26T15:34:59.700+0000 7f52c61e2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:34:59.925 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:34:59 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0[31967]: 2024-04-26T15:34:59.700+0000 7f52c61e2700 -1 osd.0 73 *** Got signal Terminated *** 2024-04-26T15:34:59.925 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:34:59 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0[31967]: 2024-04-26T15:34:59.700+0000 7f52c61e2700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:35:00.181 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:00 smithi112 ceph-mon[63158]: osd.0 marked itself down and dead 2024-04-26T15:35:00.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:00 smithi164 ceph-mon[49952]: osd.0 marked itself down and dead 2024-04-26T15:35:00.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 180026 11.39 MB/sec execute 264 sec latency 772.234 ms 2024-04-26T15:35:00.916 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:00 smithi112 podman[68688]: 2024-04-26 15:35:00.627516025 +0000 UTC m=+1.074898067 container died 72c22bffb3c9cd99bd531059a89a3ab5838e7b7857fd27e6de9d0ea1fda326cc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, ceph=True, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.openshift.expose-services=, vendor=Red Hat, Inc., name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, vcs-type=git, release=754, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, version=8, io.buildah.version=1.19.8) 2024-04-26T15:35:01.175 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:01 smithi112 ceph-mon[63158]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:35:01.176 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:01 smithi112 ceph-mon[63158]: osdmap e74: 8 total, 7 up, 8 in 2024-04-26T15:35:01.176 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:01 smithi112 ceph-mon[63158]: pgmap v60: 97 pgs: 13 stale+active+clean, 84 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.6 MiB/s wr, 638 op/s 2024-04-26T15:35:01.176 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:01 smithi112 podman[68688]: 2024-04-26 15:35:01.016730399 +0000 UTC m=+1.464112429 container cleanup 72c22bffb3c9cd99bd531059a89a3ab5838e7b7857fd27e6de9d0ea1fda326cc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, distribution-scope=public, RELEASE=HEAD, version=8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vcs-type=git, release=754, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-26T15:35:01.176 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:01 smithi112 bash[68688]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0 2024-04-26T15:35:01.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:01 smithi164 ceph-mon[49952]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:35:01.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:01 smithi164 ceph-mon[49952]: osdmap e74: 8 total, 7 up, 8 in 2024-04-26T15:35:01.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:01 smithi164 ceph-mon[49952]: pgmap v60: 97 pgs: 13 stale+active+clean, 84 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.6 MiB/s wr, 638 op/s 2024-04-26T15:35:01.429 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:01 smithi112 podman[68705]: 2024-04-26 15:35:01.318750217 +0000 UTC m=+0.688332789 container remove 72c22bffb3c9cd99bd531059a89a3ab5838e7b7857fd27e6de9d0ea1fda326cc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, name=centos-stream, maintainer=Guillaume Abrioux , version=8, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., release=754) 2024-04-26T15:35:01.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 180348 11.40 MB/sec execute 265 sec latency 1038.273 ms 2024-04-26T15:35:01.939 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:01 smithi112 podman[68819]: 2024-04-26 15:35:01.656993612 +0000 UTC m=+0.025120837 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:35:02.191 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:01 smithi112 podman[68819]: 2024-04-26 15:35:01.962522208 +0000 UTC m=+0.330649414 container create ff9628d3970e30db4c2c3b3012ad83a1f095d85a35f2172fb5b47d4c465760c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-deactivate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-26T15:35:02.192 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:02 smithi112 ceph-mon[63158]: osdmap e75: 8 total, 7 up, 8 in 2024-04-26T15:35:02.442 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:02 smithi112 podman[68819]: 2024-04-26 15:35:02.363682657 +0000 UTC m=+0.731809866 container init ff9628d3970e30db4c2c3b3012ad83a1f095d85a35f2172fb5b47d4c465760c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:35:02.442 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:02 smithi112 podman[68819]: 2024-04-26 15:35:02.368897259 +0000 UTC m=+0.737024468 container start ff9628d3970e30db4c2c3b3012ad83a1f095d85a35f2172fb5b47d4c465760c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T15:35:02.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:02 smithi164 ceph-mon[49952]: osdmap e75: 8 total, 7 up, 8 in 2024-04-26T15:35:02.693 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:02 smithi112 podman[68819]: 2024-04-26 15:35:02.55526545 +0000 UTC m=+0.923392677 container attach ff9628d3970e30db4c2c3b3012ad83a1f095d85a35f2172fb5b47d4c465760c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-deactivate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-26T15:35:02.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 180874 11.42 MB/sec execute 266 sec latency 31.756 ms 2024-04-26T15:35:02.944 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:02 smithi112 podman[68819]: 2024-04-26 15:35:02.801879956 +0000 UTC m=+1.170007171 container died ff9628d3970e30db4c2c3b3012ad83a1f095d85a35f2172fb5b47d4c465760c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:35:03.198 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:03 smithi112 ceph-mon[63158]: pgmap v62: 97 pgs: 13 stale+active+clean, 84 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 743 op/s 2024-04-26T15:35:03.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:03 smithi164 ceph-mon[49952]: pgmap v62: 97 pgs: 13 stale+active+clean, 84 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 743 op/s 2024-04-26T15:35:03.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 181175 11.39 MB/sec execute 267 sec latency 40.572 ms 2024-04-26T15:35:04.230 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:04 smithi112 podman[68848]: 2024-04-26 15:35:04.057783592 +0000 UTC m=+1.249976791 container remove ff9628d3970e30db4c2c3b3012ad83a1f095d85a35f2172fb5b47d4c465760c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-26T15:35:04.230 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:04 smithi112 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-26T15:35:04.231 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:04 smithi112 systemd[1]: Stopped Ceph osd.0 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:35:04.231 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:04 smithi112 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.0.service: Consumed 3min 566ms CPU time. 2024-04-26T15:35:04.490 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:04 smithi112 systemd[1]: Starting Ceph osd.0 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:35:04.741 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:04 smithi112 podman[68940]: 2024-04-26 15:35:04.618788899 +0000 UTC m=+0.023892267 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:35:04.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 181728 11.40 MB/sec execute 268 sec latency 43.355 ms 2024-04-26T15:35:05.001 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:04 smithi112 podman[68940]: 2024-04-26 15:35:04.745408527 +0000 UTC m=+0.150511886 container create 37f9fcd7d96f878a46372504154a0a290660a778ec7f2fdecf00eee2fda13525 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-26T15:35:05.261 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:05 smithi112 ceph-mon[63158]: pgmap v63: 97 pgs: 7 active+undersized+degraded, 1 active+undersized, 77 active+clean, 12 stale+active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.2 MiB/s wr, 425 op/s; 50/1707 objects degraded (2.929%) 2024-04-26T15:35:05.261 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:05 smithi112 ceph-mon[63158]: Health check failed: Degraded data redundancy: 50/1707 objects degraded (2.929%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T15:35:05.262 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 podman[68940]: 2024-04-26 15:35:05.096846911 +0000 UTC m=+0.501950273 container init 37f9fcd7d96f878a46372504154a0a290660a778ec7f2fdecf00eee2fda13525 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-26T15:35:05.262 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 podman[68940]: 2024-04-26 15:35:05.10445008 +0000 UTC m=+0.509553440 container start 37f9fcd7d96f878a46372504154a0a290660a778ec7f2fdecf00eee2fda13525 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-26T15:35:05.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:05 smithi164 ceph-mon[49952]: pgmap v63: 97 pgs: 7 active+undersized+degraded, 1 active+undersized, 77 active+clean, 12 stale+active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.2 MiB/s wr, 425 op/s; 50/1707 objects degraded (2.929%) 2024-04-26T15:35:05.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:05 smithi164 ceph-mon[49952]: Health check failed: Degraded data redundancy: 50/1707 objects degraded (2.929%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T15:35:05.512 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 podman[68940]: 2024-04-26 15:35:05.282213801 +0000 UTC m=+0.687317171 container attach 37f9fcd7d96f878a46372504154a0a290660a778ec7f2fdecf00eee2fda13525 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-26T15:35:05.764 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate[68955]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:35:05.764 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate[68955]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:35:05.764 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 bash[68940]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:35:05.764 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 bash[68940]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:35:05.764 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 bash[68940]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T15:35:05.764 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 bash[68940]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T15:35:05.764 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 bash[68940]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T15:35:05.764 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 bash[68940]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T15:35:05.764 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 bash[68940]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T15:35:05.765 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 bash[68940]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T15:35:05.765 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 bash[68940]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T15:35:05.765 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate[68955]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T15:35:05.765 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate[68955]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T15:35:05.765 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate[68955]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T15:35:05.765 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate[68955]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T15:35:05.765 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate[68955]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T15:35:05.765 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate[68955]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T15:35:05.765 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate[68955]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T15:35:05.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 182188 11.40 MB/sec execute 269 sec latency 60.935 ms 2024-04-26T15:35:06.015 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:05 smithi112 podman[68940]: 2024-04-26 15:35:05.780258286 +0000 UTC m=+1.185361651 container died 37f9fcd7d96f878a46372504154a0a290660a778ec7f2fdecf00eee2fda13525 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True) 2024-04-26T15:35:06.792 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:06 smithi112 podman[69093]: 2024-04-26 15:35:06.76806445 +0000 UTC m=+0.981602921 container remove 37f9fcd7d96f878a46372504154a0a290660a778ec7f2fdecf00eee2fda13525 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-26T15:35:06.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 182521 11.39 MB/sec execute 270 sec latency 57.588 ms 2024-04-26T15:35:07.046 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:07 smithi112 podman[69145]: 2024-04-26 15:35:06.923891932 +0000 UTC m=+0.027098089 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:35:07.047 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:07 smithi112 podman[69145]: 2024-04-26 15:35:07.035975114 +0000 UTC m=+0.139181253 container create ed1d8391102ff5ad9df5a1ee94bd79bde3da2872abf55d99895d97574e39ffec (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-26T15:35:07.310 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:07 smithi112 ceph-mon[63158]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 355 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.3 MiB/s wr, 740 op/s; 233/1722 objects degraded (13.531%) 2024-04-26T15:35:07.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:07 smithi164 ceph-mon[49952]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 355 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.3 MiB/s wr, 740 op/s; 233/1722 objects degraded (13.531%) 2024-04-26T15:35:07.562 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:07 smithi112 podman[69145]: 2024-04-26 15:35:07.350807911 +0000 UTC m=+0.454014056 container init ed1d8391102ff5ad9df5a1ee94bd79bde3da2872abf55d99895d97574e39ffec (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T15:35:07.563 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:07 smithi112 podman[69145]: 2024-04-26 15:35:07.36167103 +0000 UTC m=+0.464877171 container start ed1d8391102ff5ad9df5a1ee94bd79bde3da2872abf55d99895d97574e39ffec (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-26T15:35:07.563 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:07 smithi112 ceph-osd[69165]: -- 172.21.15.112:0/2641501327 <== mon.0 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c203ebf680 con 0x55c203e89000 2024-04-26T15:35:07.813 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:07 smithi112 bash[69145]: ed1d8391102ff5ad9df5a1ee94bd79bde3da2872abf55d99895d97574e39ffec 2024-04-26T15:35:07.813 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:07 smithi112 systemd[1]: Started Ceph osd.0 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:35:07.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 182945 11.37 MB/sec execute 271 sec latency 53.201 ms 2024-04-26T15:35:08.064 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:07 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0[69161]: 2024-04-26T15:35:07.953+0000 7f52402bf7c0 -1 Falling back to public interface 2024-04-26T15:35:08.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:08.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:08.849 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:35:08.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 183185 11.35 MB/sec execute 272 sec latency 38.134 ms 2024-04-26T15:35:09.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:09.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:09.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:35:09.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 183449 11.35 MB/sec execute 273 sec latency 47.137 ms 2024-04-26T15:35:09.871 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:09 smithi112 ceph-mon[63158]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 355 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.3 MiB/s wr, 302 op/s; 233/1722 objects degraded (13.531%) 2024-04-26T15:35:09.872 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:09.872 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:35:10.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:09 smithi164 ceph-mon[49952]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 355 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.3 MiB/s wr, 302 op/s; 233/1722 objects degraded (13.531%) 2024-04-26T15:35:10.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:10.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:35:10.534 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:35:10.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 183907 11.37 MB/sec execute 274 sec latency 61.743 ms 2024-04-26T15:35:11.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:11 smithi112 ceph-mon[63158]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:11.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:11 smithi112 ceph-mon[63158]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.3 MiB/s wr, 527 op/s; 211/1578 objects degraded (13.371%) 2024-04-26T15:35:11.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:11 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 211/1578 objects degraded (13.371%), 29 pgs degraded (PG_DEGRADED) 2024-04-26T15:35:11.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:11 smithi164 ceph-mon[49952]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:11.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:11 smithi164 ceph-mon[49952]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.3 MiB/s wr, 527 op/s; 211/1578 objects degraded (13.371%) 2024-04-26T15:35:11.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:11 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 211/1578 objects degraded (13.371%), 29 pgs degraded (PG_DEGRADED) 2024-04-26T15:35:11.547 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:35:11.547 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (4m) 29s ago 13m 26.1M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (45s) 29s ago 13m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (38s) 34s ago 12m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (3m) 29s ago 12m 53.4M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (7m) 29s ago 7m 37.8M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (7m) 34s ago 7m 14.0M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (3m) 29s ago 14m 511M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (2m) 34s ago 12m 465M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (2m) 29s ago 14m 42.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (104s) 34s ago 12m 35.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (6m) 29s ago 6m 251M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (6m) 34s ago 6m 87.6M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (3m) 29s ago 12m 17.6M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (3m) 34s ago 12m 17.6M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 starting - - - 1936M 2024-04-26T15:35:11.548 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (11m) 29s ago 11m 526M 1936M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:35:11.549 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (10m) 29s ago 10m 413M 1936M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:35:11.549 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (10m) 29s ago 10m 514M 1936M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:35:11.549 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (9m) 34s ago 9m 551M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:35:11.549 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (9m) 34s ago 9m 373M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:35:11.549 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (9m) 34s ago 9m 557M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:35:11.549 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (8m) 34s ago 8m 448M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:35:11.549 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (2m) 29s ago 11m 62.7M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:35:11.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 184349 11.38 MB/sec execute 275 sec latency 38.219 ms 2024-04-26T15:35:12.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:12 smithi112 ceph-mon[63158]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:12.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:12 smithi112 ceph-mon[63158]: from='client.44147 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:12.178 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:35:12.178 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:35:12.178 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:35:12.178 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:35:12.178 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:35:12.178 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:35:12.178 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:35:12.178 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:35:12.178 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-04-26T15:35:12.178 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:35:12.178 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:35:12.178 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:35:12.179 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:35:12.179 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:35:12.179 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-26T15:35:12.179 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 4 2024-04-26T15:35:12.179 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:35:12.179 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:35:12.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:12 smithi164 ceph-mon[49952]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:12.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:12 smithi164 ceph-mon[49952]: from='client.44147 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout: "crash", 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout: "mon" 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "6/23 daemons upgraded", 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:35:12.711 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:35:12.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 184655 11.37 MB/sec execute 276 sec latency 34.367 ms 2024-04-26T15:35:13.188 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:13 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/3673162207' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:35:13.189 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:13 smithi112 ceph-mon[63158]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:13.189 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:13 smithi112 ceph-mon[63158]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 443 op/s; 211/1578 objects degraded (13.371%) 2024-04-26T15:35:13.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:13 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/3673162207' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:35:13.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:13 smithi164 ceph-mon[49952]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:13.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:13 smithi164 ceph-mon[49952]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 443 op/s; 211/1578 objects degraded (13.371%) 2024-04-26T15:35:13.385 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 211/1578 objects degraded (13.371%), 29 pgs degraded 2024-04-26T15:35:13.385 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-26T15:35:13.385 INFO:teuthology.orchestra.run.smithi112.stdout: osd.0 (root=default,host=smithi112) is down 2024-04-26T15:35:13.385 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 211/1578 objects degraded (13.371%), 29 pgs degraded 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.5 is active+undersized+degraded, acting [2,4] 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-04-26T15:35:13.386 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-04-26T15:35:13.387 INFO:teuthology.orchestra.run.smithi112.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-04-26T15:35:13.388 INFO:teuthology.orchestra.run.smithi112.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-04-26T15:35:13.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 184939 11.35 MB/sec execute 277 sec latency 65.484 ms 2024-04-26T15:35:14.196 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:14 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/976465187' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:35:14.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:14 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/976465187' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:35:14.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 185259 11.33 MB/sec execute 278 sec latency 50.628 ms 2024-04-26T15:35:15.242 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:15 smithi112 ceph-mon[63158]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 348 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 465 op/s; 218/1617 objects degraded (13.482%) 2024-04-26T15:35:15.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:15 smithi164 ceph-mon[49952]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 348 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 465 op/s; 218/1617 objects degraded (13.482%) 2024-04-26T15:35:15.749 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:15 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0[69161]: 2024-04-26T15:35:15.502+0000 7f52402bf7c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-26T15:35:15.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 185533 11.31 MB/sec execute 279 sec latency 68.051 ms 2024-04-26T15:35:16.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 185934 11.32 MB/sec execute 280 sec latency 30.647 ms 2024-04-26T15:35:17.023 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:16 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0[69161]: 2024-04-26T15:35:16.857+0000 7f52402bf7c0 -1 osd.0 73 log_to_monitors true 2024-04-26T15:35:17.787 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:17 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:17.787 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:17 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:17.787 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:17 smithi112 ceph-mon[63158]: from='osd.0 [v2:172.21.15.112:6802/3769332406,v1:172.21.15.112:6803/3769332406]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T15:35:17.787 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:17 smithi112 ceph-mon[63158]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 351 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 641 op/s; 227/1701 objects degraded (13.345%) 2024-04-26T15:35:17.861 INFO:teuthology.orchestra.run.smithi112.stdout: 5 186517 11.31 MB/sec execute 281 sec latency 30.705 ms 2024-04-26T15:35:18.038 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:35:17 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0[69161]: 2024-04-26T15:35:17.821+0000 7f5231717700 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:35:18.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:17 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:18.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:17 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:18.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:17 smithi164 ceph-mon[49952]: from='osd.0 [v2:172.21.15.112:6802/3769332406,v1:172.21.15.112:6803/3769332406]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T15:35:18.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:17 smithi164 ceph-mon[49952]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 351 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 641 op/s; 227/1701 objects degraded (13.345%) 2024-04-26T15:35:18.791 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:18 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 227/1701 objects degraded (13.345%), 29 pgs degraded (PG_DEGRADED) 2024-04-26T15:35:18.792 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:18 smithi112 ceph-mon[63158]: from='osd.0 [v2:172.21.15.112:6802/3769332406,v1:172.21.15.112:6803/3769332406]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T15:35:18.792 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:18 smithi112 ceph-mon[63158]: osdmap e76: 8 total, 7 up, 8 in 2024-04-26T15:35:18.792 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:18 smithi112 ceph-mon[63158]: from='osd.0 [v2:172.21.15.112:6802/3769332406,v1:172.21.15.112:6803/3769332406]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:35:18.792 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:18 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:18.792 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:18 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:18.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 186850 11.30 MB/sec execute 282 sec latency 75.674 ms 2024-04-26T15:35:19.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:18 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 227/1701 objects degraded (13.345%), 29 pgs degraded (PG_DEGRADED) 2024-04-26T15:35:19.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:18 smithi164 ceph-mon[49952]: from='osd.0 [v2:172.21.15.112:6802/3769332406,v1:172.21.15.112:6803/3769332406]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T15:35:19.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:18 smithi164 ceph-mon[49952]: osdmap e76: 8 total, 7 up, 8 in 2024-04-26T15:35:19.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:18 smithi164 ceph-mon[49952]: from='osd.0 [v2:172.21.15.112:6802/3769332406,v1:172.21.15.112:6803/3769332406]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:35:19.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:18 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:19.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:18 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:19.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 186880 11.26 MB/sec execute 283 sec latency 1052.287 ms 2024-04-26T15:35:19.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:19 smithi112 ceph-mon[63158]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:35:19.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:19 smithi112 ceph-mon[63158]: osd.0 [v2:172.21.15.112:6802/3769332406,v1:172.21.15.112:6803/3769332406] boot 2024-04-26T15:35:19.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:19 smithi112 ceph-mon[63158]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T15:35:19.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:19 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:35:19.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:19 smithi112 ceph-mon[63158]: pgmap v72: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.5 MiB/s wr, 325 op/s; 227/1701 objects degraded (13.345%) 2024-04-26T15:35:19.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:19 smithi112 ceph-mon[63158]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T15:35:20.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:19 smithi164 ceph-mon[49952]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:35:20.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:19 smithi164 ceph-mon[49952]: osd.0 [v2:172.21.15.112:6802/3769332406,v1:172.21.15.112:6803/3769332406] boot 2024-04-26T15:35:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:19 smithi164 ceph-mon[49952]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T15:35:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:19 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:35:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:19 smithi164 ceph-mon[49952]: pgmap v72: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.5 MiB/s wr, 325 op/s; 227/1701 objects degraded (13.345%) 2024-04-26T15:35:20.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:19 smithi164 ceph-mon[49952]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T15:35:20.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 186906 11.23 MB/sec execute 284 sec latency 2028.516 ms 2024-04-26T15:35:21.481 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:21 smithi112 ceph-mon[63158]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T15:35:21.481 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:21 smithi112 ceph-mon[63158]: pgmap v75: 97 pgs: 12 remapped+peering, 3 active+undersized+degraded, 5 active+undersized, 77 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.7 MiB/s wr, 591 op/s; 25/1722 objects degraded (1.452%); 22 B/s, 5 objects/s recovering 2024-04-26T15:35:21.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:21 smithi164 ceph-mon[49952]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T15:35:21.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:21 smithi164 ceph-mon[49952]: pgmap v75: 97 pgs: 12 remapped+peering, 3 active+undersized+degraded, 5 active+undersized, 77 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.7 MiB/s wr, 591 op/s; 25/1722 objects degraded (1.452%); 22 B/s, 5 objects/s recovering 2024-04-26T15:35:21.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 187368 11.26 MB/sec execute 285 sec latency 387.464 ms 2024-04-26T15:35:22.506 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:22 smithi112 ceph-mon[63158]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T15:35:22.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:22 smithi164 ceph-mon[49952]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T15:35:22.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 187770 11.26 MB/sec execute 286 sec latency 847.578 ms 2024-04-26T15:35:23.438 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T15:35:23.438 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: pgmap v78: 97 pgs: 11 remapped+peering, 86 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.9 MiB/s wr, 606 op/s; 22 B/s, 17 objects/s recovering 2024-04-26T15:35:23.438 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:23.438 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:23.438 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:35:23.439 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:35:23.439 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:23.439 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:23.439 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:35:23.439 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:35:23.439 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:35:23.439 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:35:23.439 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T15:35:23.519 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T15:35:23.519 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: pgmap v78: 97 pgs: 11 remapped+peering, 86 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.9 MiB/s wr, 606 op/s; 22 B/s, 17 objects/s recovering 2024-04-26T15:35:23.519 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:23.519 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:23.519 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:35:23.519 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:35:23.520 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:23.520 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:23.520 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:35:23.520 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:35:23.520 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:35:23.520 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:35:23.520 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T15:35:23.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 188021 11.24 MB/sec execute 287 sec latency 685.500 ms 2024-04-26T15:35:24.567 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:24 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T15:35:24.567 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:24 smithi112 ceph-mon[63158]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T15:35:24.567 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:24 smithi112 ceph-mon[63158]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 25/1722 objects degraded (1.452%), 3 pgs degraded) 2024-04-26T15:35:24.567 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:24 smithi112 ceph-mon[63158]: Cluster is now healthy 2024-04-26T15:35:24.567 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:24 smithi112 ceph-mon[63158]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T15:35:24.567 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:24.567 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:35:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:24 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T15:35:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:24 smithi164 ceph-mon[49952]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T15:35:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:24 smithi164 ceph-mon[49952]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 25/1722 objects degraded (1.452%), 3 pgs degraded) 2024-04-26T15:35:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:24 smithi164 ceph-mon[49952]: Cluster is now healthy 2024-04-26T15:35:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:24 smithi164 ceph-mon[49952]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T15:35:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:24.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:35:24.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 188262 11.22 MB/sec execute 288 sec latency 1026.107 ms 2024-04-26T15:35:25.323 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:25 smithi112 ceph-mon[63158]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T15:35:25.323 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:25 smithi112 ceph-mon[63158]: pgmap v81: 97 pgs: 6 remapped+peering, 91 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.8 MiB/s wr, 454 op/s; 1024 KiB/s, 35 objects/s recovering 2024-04-26T15:35:25.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:25 smithi164 ceph-mon[49952]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T15:35:25.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:25 smithi164 ceph-mon[49952]: pgmap v81: 97 pgs: 6 remapped+peering, 91 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.8 MiB/s wr, 454 op/s; 1024 KiB/s, 35 objects/s recovering 2024-04-26T15:35:25.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 188734 11.24 MB/sec execute 289 sec latency 28.201 ms 2024-04-26T15:35:26.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 189160 11.22 MB/sec execute 290 sec latency 49.898 ms 2024-04-26T15:35:27.099 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:27 smithi112 ceph-mon[63158]: pgmap v82: 97 pgs: 4 remapped+peering, 93 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.5 MiB/s wr, 350 op/s; 1.4 MiB/s, 29 objects/s recovering 2024-04-26T15:35:27.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:27 smithi164 ceph-mon[49952]: pgmap v82: 97 pgs: 4 remapped+peering, 93 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.5 MiB/s wr, 350 op/s; 1.4 MiB/s, 29 objects/s recovering 2024-04-26T15:35:27.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 189436 11.21 MB/sec execute 291 sec latency 50.568 ms 2024-04-26T15:35:28.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 189670 11.18 MB/sec execute 292 sec latency 48.220 ms 2024-04-26T15:35:29.155 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:29 smithi112 ceph-mon[63158]: pgmap v83: 97 pgs: 97 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.8 MiB/s wr, 330 op/s; 1.2 MiB/s, 41 objects/s recovering 2024-04-26T15:35:29.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:29 smithi164 ceph-mon[49952]: pgmap v83: 97 pgs: 97 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.8 MiB/s wr, 330 op/s; 1.2 MiB/s, 41 objects/s recovering 2024-04-26T15:35:29.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 189983 11.18 MB/sec execute 293 sec latency 56.260 ms 2024-04-26T15:35:30.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 190222 11.16 MB/sec execute 294 sec latency 46.161 ms 2024-04-26T15:35:31.187 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:31 smithi112 ceph-mon[63158]: pgmap v84: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 555 op/s; 1.0 MiB/s, 28 objects/s recovering 2024-04-26T15:35:31.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:31 smithi164 ceph-mon[49952]: pgmap v84: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 555 op/s; 1.0 MiB/s, 28 objects/s recovering 2024-04-26T15:35:31.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 190563 11.18 MB/sec execute 295 sec latency 52.460 ms 2024-04-26T15:35:32.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 190889 11.17 MB/sec execute 296 sec latency 43.585 ms 2024-04-26T15:35:33.133 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:33 smithi112 ceph-mon[63158]: pgmap v85: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 473 op/s; 858 KiB/s, 23 objects/s recovering 2024-04-26T15:35:33.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:33 smithi164 ceph-mon[49952]: pgmap v85: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 473 op/s; 858 KiB/s, 23 objects/s recovering 2024-04-26T15:35:33.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 191141 11.15 MB/sec execute 297 sec latency 50.453 ms 2024-04-26T15:35:34.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 191507 11.14 MB/sec execute 298 sec latency 65.853 ms 2024-04-26T15:35:35.166 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:35 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:35.166 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:35 smithi112 ceph-mon[63158]: pgmap v86: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 446 op/s; 778 KiB/s, 21 objects/s recovering 2024-04-26T15:35:35.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:35 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:35.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:35 smithi164 ceph-mon[49952]: pgmap v86: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 446 op/s; 778 KiB/s, 21 objects/s recovering 2024-04-26T15:35:35.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 191877 11.13 MB/sec execute 299 sec latency 33.139 ms 2024-04-26T15:35:36.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 192321 11.15 MB/sec execute 300 sec latency 45.898 ms 2024-04-26T15:35:37.206 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:37 smithi112 ceph-mon[63158]: pgmap v87: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 411 op/s; 352 KiB/s, 11 objects/s recovering 2024-04-26T15:35:37.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:37 smithi164 ceph-mon[49952]: pgmap v87: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 411 op/s; 352 KiB/s, 11 objects/s recovering 2024-04-26T15:35:37.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 192698 11.13 MB/sec execute 301 sec latency 54.411 ms 2024-04-26T15:35:38.484 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:38 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T15:35:38.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:38 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T15:35:38.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 193024 11.12 MB/sec execute 302 sec latency 54.167 ms 2024-04-26T15:35:39.511 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:39 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T15:35:39.511 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:39 smithi112 ceph-mon[63158]: Upgrade: osd.1 is safe to restart 2024-04-26T15:35:39.512 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:39 smithi112 ceph-mon[63158]: pgmap v88: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 409 op/s; 0 B/s, 8 objects/s recovering 2024-04-26T15:35:39.512 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:35:39.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:39 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T15:35:39.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:39 smithi164 ceph-mon[49952]: Upgrade: osd.1 is safe to restart 2024-04-26T15:35:39.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:39 smithi164 ceph-mon[49952]: pgmap v88: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 409 op/s; 0 B/s, 8 objects/s recovering 2024-04-26T15:35:39.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:35:39.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 193337 11.10 MB/sec execute 303 sec latency 33.784 ms 2024-04-26T15:35:40.863 INFO:teuthology.orchestra.run.smithi112.stdout: 5 193821 11.12 MB/sec execute 304 sec latency 42.072 ms 2024-04-26T15:35:41.291 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:41 smithi112 ceph-mon[63158]: pgmap v89: 97 pgs: 97 active+clean; 349 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.8 MiB/s wr, 604 op/s 2024-04-26T15:35:41.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:41 smithi164 ceph-mon[49952]: pgmap v89: 97 pgs: 97 active+clean; 349 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.8 MiB/s wr, 604 op/s 2024-04-26T15:35:41.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 194363 11.14 MB/sec execute 305 sec latency 35.250 ms 2024-04-26T15:35:42.555 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:42 smithi112 ceph-mon[63158]: Upgrade: Updating osd.1 2024-04-26T15:35:42.555 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:42 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:42.555 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:42 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T15:35:42.555 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:42 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:35:42.555 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:42 smithi112 ceph-mon[63158]: Deploying daemon osd.1 on smithi112 2024-04-26T15:35:42.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:42 smithi164 ceph-mon[49952]: Upgrade: Updating osd.1 2024-04-26T15:35:42.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:42 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:42.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:42 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T15:35:42.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:42 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:35:42.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:42 smithi164 ceph-mon[49952]: Deploying daemon osd.1 on smithi112 2024-04-26T15:35:42.863 INFO:teuthology.orchestra.run.smithi112.stdout: 5 194856 11.14 MB/sec execute 306 sec latency 45.280 ms 2024-04-26T15:35:43.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:43 smithi112 ceph-mon[63158]: pgmap v90: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 430 op/s 2024-04-26T15:35:43.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:43 smithi164 ceph-mon[49952]: pgmap v90: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 430 op/s 2024-04-26T15:35:43.862 INFO:teuthology.orchestra.run.smithi112.stdout: 5 195336 11.14 MB/sec execute 307 sec latency 34.887 ms 2024-04-26T15:35:43.950 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:35:44.574 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:44 smithi112 ceph-mon[63158]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:44.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:44 smithi164 ceph-mon[49952]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:44.863 INFO:teuthology.orchestra.run.smithi112.stdout: 5 195750 11.15 MB/sec execute 308 sec latency 43.628 ms 2024-04-26T15:35:45.120 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:35:45.120 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (4m) 28s ago 13m 26.1M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (78s) 28s ago 13m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (72s) 67s ago 13m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (4m) 28s ago 13m 53.9M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (7m) 28s ago 7m 37.9M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (7m) 67s ago 7m 14.0M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (4m) 28s ago 14m 513M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (3m) 67s ago 12m 465M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (2m) 28s ago 15m 46.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (2m) 67s ago 12m 35.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (7m) 28s ago 7m 257M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (7m) 67s ago 7m 87.6M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (4m) 28s ago 13m 17.8M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:35:45.121 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (4m) 67s ago 12m 17.6M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:35:45.122 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (38s) 28s ago 12m 12.1M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:35:45.122 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (11m) 28s ago 11m 569M 1936M 17.2.0 e1d6a67b021e 65ea07f9ef08 2024-04-26T15:35:45.122 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (11m) 28s ago 11m 457M 1936M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:35:45.122 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (10m) 28s ago 10m 560M 1936M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:35:45.122 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (10m) 67s ago 10m 551M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:35:45.122 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (9m) 67s ago 9m 373M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:35:45.122 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (9m) 67s ago 9m 557M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:35:45.122 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (9m) 67s ago 9m 448M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:35:45.122 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (3m) 28s ago 12m 67.7M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:35:45.596 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:45 smithi112 ceph-mon[63158]: from='client.44165 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:45.596 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:45 smithi112 ceph-mon[63158]: pgmap v91: 97 pgs: 97 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.3 MiB/s wr, 557 op/s 2024-04-26T15:35:45.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:45 smithi164 ceph-mon[49952]: from='client.44165 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:45.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:45 smithi164 ceph-mon[49952]: pgmap v91: 97 pgs: 97 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.3 MiB/s wr, 557 op/s 2024-04-26T15:35:45.837 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:35:45.837 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:35:45.837 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:35:45.837 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:35:45.837 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:35:45.837 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 1 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 5 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:35:45.838 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:35:45.863 INFO:teuthology.orchestra.run.smithi112.stdout: 5 196160 11.15 MB/sec execute 309 sec latency 39.948 ms 2024-04-26T15:35:46.348 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:46 smithi112 ceph-mon[63158]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:46.348 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:46 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/1502247127' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:35:46.480 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:35:46.480 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:35:46.480 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:35:46.480 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:35:46.481 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:35:46.481 INFO:teuthology.orchestra.run.smithi112.stdout: "crash", 2024-04-26T15:35:46.481 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:35:46.481 INFO:teuthology.orchestra.run.smithi112.stdout: "mon" 2024-04-26T15:35:46.481 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:35:46.481 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "7/23 daemons upgraded", 2024-04-26T15:35:46.481 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T15:35:46.481 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:35:46.481 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:35:46.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:46 smithi164 ceph-mon[49952]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:46.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:46 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/1502247127' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:35:46.864 INFO:teuthology.orchestra.run.smithi112.stdout: 5 196554 11.14 MB/sec execute 310 sec latency 54.118 ms 2024-04-26T15:35:47.104 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:46 smithi112 systemd[1]: Stopping Ceph osd.1 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:35:47.172 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:35:47.354 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:47 smithi112 ceph-mon[63158]: from='client.44177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:47.354 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:47 smithi112 ceph-mon[63158]: pgmap v92: 97 pgs: 97 active+clean; 359 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.6 MiB/s wr, 674 op/s 2024-04-26T15:35:47.354 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:47 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/4160372606' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:35:47.612 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:47 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[35107]: 2024-04-26T15:35:47.389+0000 7f2452b46700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:35:47.612 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:47 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[35107]: 2024-04-26T15:35:47.389+0000 7f2452b46700 -1 osd.1 83 *** Got signal Terminated *** 2024-04-26T15:35:47.612 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:47 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[35107]: 2024-04-26T15:35:47.389+0000 7f2452b46700 -1 osd.1 83 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:35:47.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:47 smithi164 ceph-mon[49952]: from='client.44177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:35:47.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:47 smithi164 ceph-mon[49952]: pgmap v92: 97 pgs: 97 active+clean; 359 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.6 MiB/s wr, 674 op/s 2024-04-26T15:35:47.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:47 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/4160372606' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:35:47.864 INFO:teuthology.orchestra.run.smithi112.stdout: 5 196941 11.13 MB/sec execute 311 sec latency 27.470 ms 2024-04-26T15:35:48.391 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:48 smithi112 ceph-mon[63158]: osd.1 marked itself down and dead 2024-04-26T15:35:48.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:48 smithi164 ceph-mon[49952]: osd.1 marked itself down and dead 2024-04-26T15:35:48.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 197231 11.12 MB/sec execute 312 sec latency 509.022 ms 2024-04-26T15:35:49.259 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:48 smithi112 podman[71418]: 2024-04-26 15:35:48.977089172 +0000 UTC m=+1.740994703 container died 65ea07f9ef085ab0e5902e7bd8c2164b628efd010d058a95fa56e460e32bfa9f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, architecture=x86_64, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-26T15:35:49.513 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:49 smithi112 ceph-mon[63158]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:35:49.513 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:49 smithi112 ceph-mon[63158]: osdmap e84: 8 total, 7 up, 8 in 2024-04-26T15:35:49.513 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:49 smithi112 ceph-mon[63158]: pgmap v94: 97 pgs: 14 stale+active+clean, 83 active+clean; 358 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.8 MiB/s wr, 730 op/s 2024-04-26T15:35:49.514 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:49 smithi112 podman[71418]: 2024-04-26 15:35:49.367182275 +0000 UTC m=+2.131087803 container cleanup 65ea07f9ef085ab0e5902e7bd8c2164b628efd010d058a95fa56e460e32bfa9f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1, com.redhat.component=centos-stream-container, release=754, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, version=8, RELEASE=HEAD, name=centos-stream, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-26T15:35:49.514 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:49 smithi112 bash[71418]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1 2024-04-26T15:35:49.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:49 smithi164 ceph-mon[49952]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:35:49.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:49 smithi164 ceph-mon[49952]: osdmap e84: 8 total, 7 up, 8 in 2024-04-26T15:35:49.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:49 smithi164 ceph-mon[49952]: pgmap v94: 97 pgs: 14 stale+active+clean, 83 active+clean; 358 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.8 MiB/s wr, 730 op/s 2024-04-26T15:35:49.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 197410 11.11 MB/sec execute 313 sec latency 1034.696 ms 2024-04-26T15:35:50.270 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:50 smithi112 podman[71433]: 2024-04-26 15:35:50.054106223 +0000 UTC m=+1.081469732 container remove 65ea07f9ef085ab0e5902e7bd8c2164b628efd010d058a95fa56e460e32bfa9f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, release=754, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.k8s.display-name=CentOS Stream 8, version=8, maintainer=Guillaume Abrioux , io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.buildah.version=1.19.8) 2024-04-26T15:35:50.522 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:50 smithi112 ceph-mon[63158]: osdmap e85: 8 total, 7 up, 8 in 2024-04-26T15:35:50.776 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:50 smithi112 podman[71517]: 2024-04-26 15:35:50.501864275 +0000 UTC m=+0.024187392 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:35:50.776 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:50 smithi112 podman[71517]: 2024-04-26 15:35:50.690690666 +0000 UTC m=+0.213013777 container create 8b968dadcb67c7e353e0e5e18dc022f80754eed99dc81e7398f650ff88464462 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-26T15:35:50.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:50 smithi164 ceph-mon[49952]: osdmap e85: 8 total, 7 up, 8 in 2024-04-26T15:35:50.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 197817 11.12 MB/sec execute 314 sec latency 75.724 ms 2024-04-26T15:35:51.033 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:51 smithi112 podman[71517]: 2024-04-26 15:35:51.01801156 +0000 UTC m=+0.540334669 container init 8b968dadcb67c7e353e0e5e18dc022f80754eed99dc81e7398f650ff88464462 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T15:35:51.033 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:51 smithi112 podman[71517]: 2024-04-26 15:35:51.023557856 +0000 UTC m=+0.545880973 container start 8b968dadcb67c7e353e0e5e18dc022f80754eed99dc81e7398f650ff88464462 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS) 2024-04-26T15:35:51.288 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:51 smithi112 podman[71517]: 2024-04-26 15:35:51.191697294 +0000 UTC m=+0.714020405 container attach 8b968dadcb67c7e353e0e5e18dc022f80754eed99dc81e7398f650ff88464462 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, ceph=True) 2024-04-26T15:35:51.546 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:51 smithi112 ceph-mon[63158]: pgmap v96: 97 pgs: 22 active+undersized+degraded, 1 stale+active+clean, 13 active+undersized, 61 active+clean; 346 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 726 op/s; 160/1620 objects degraded (9.877%) 2024-04-26T15:35:51.546 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:51 smithi112 podman[71517]: 2024-04-26 15:35:51.468209736 +0000 UTC m=+0.990532849 container died 8b968dadcb67c7e353e0e5e18dc022f80754eed99dc81e7398f650ff88464462 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-26T15:35:51.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:51 smithi164 ceph-mon[49952]: pgmap v96: 97 pgs: 22 active+undersized+degraded, 1 stale+active+clean, 13 active+undersized, 61 active+clean; 346 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 726 op/s; 160/1620 objects degraded (9.877%) 2024-04-26T15:35:51.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 198189 11.11 MB/sec execute 315 sec latency 54.876 ms 2024-04-26T15:35:52.563 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:52 smithi112 podman[71543]: 2024-04-26 15:35:52.399869923 +0000 UTC m=+0.917303885 container remove 8b968dadcb67c7e353e0e5e18dc022f80754eed99dc81e7398f650ff88464462 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-26T15:35:52.563 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:52 smithi112 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-26T15:35:52.563 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:52 smithi112 systemd[1]: Stopped Ceph osd.1 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:35:52.563 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:52 smithi112 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.1.service: Consumed 3min 34.803s CPU time. 2024-04-26T15:35:52.563 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:52 smithi112 ceph-mon[63158]: Health check failed: Degraded data redundancy: 160/1620 objects degraded (9.877%), 22 pgs degraded (PG_DEGRADED) 2024-04-26T15:35:52.748 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:52 smithi164 ceph-mon[49952]: Health check failed: Degraded data redundancy: 160/1620 objects degraded (9.877%), 22 pgs degraded (PG_DEGRADED) 2024-04-26T15:35:52.813 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:52 smithi112 systemd[1]: Starting Ceph osd.1 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:35:52.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 198499 11.10 MB/sec execute 316 sec latency 39.780 ms 2024-04-26T15:35:53.068 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:52 smithi112 podman[71643]: 2024-04-26 15:35:52.895944699 +0000 UTC m=+0.022633775 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:35:53.068 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:53 smithi112 podman[71643]: 2024-04-26 15:35:53.028617921 +0000 UTC m=+0.155306964 container create 5f4999ed9af80c236dde75e3f13829d2a75705b74a93157713067386bc0484c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T15:35:53.573 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:53 smithi112 podman[71643]: 2024-04-26 15:35:53.38452237 +0000 UTC m=+0.511211425 container init 5f4999ed9af80c236dde75e3f13829d2a75705b74a93157713067386bc0484c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-26T15:35:53.574 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:53 smithi112 podman[71643]: 2024-04-26 15:35:53.39454264 +0000 UTC m=+0.521231683 container start 5f4999ed9af80c236dde75e3f13829d2a75705b74a93157713067386bc0484c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-26T15:35:53.574 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:53 smithi112 podman[71643]: 2024-04-26 15:35:53.558376687 +0000 UTC m=+0.685065737 container attach 5f4999ed9af80c236dde75e3f13829d2a75705b74a93157713067386bc0484c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-26T15:35:53.574 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:53 smithi112 ceph-mon[63158]: pgmap v97: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 542 op/s; 181/1650 objects degraded (10.970%) 2024-04-26T15:35:53.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:53 smithi164 ceph-mon[49952]: pgmap v97: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 542 op/s; 181/1650 objects degraded (10.970%) 2024-04-26T15:35:53.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 198870 11.10 MB/sec execute 317 sec latency 36.942 ms 2024-04-26T15:35:54.103 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate[71657]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:35:54.103 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate[71657]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:35:54.103 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 bash[71643]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:35:54.103 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 bash[71643]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:35:54.103 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 bash[71643]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T15:35:54.103 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 bash[71643]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T15:35:54.103 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 bash[71643]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T15:35:54.103 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 bash[71643]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T15:35:54.103 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 bash[71643]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T15:35:54.103 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 bash[71643]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T15:35:54.103 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 bash[71643]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T15:35:54.104 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate[71657]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T15:35:54.104 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate[71657]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T15:35:54.104 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate[71657]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T15:35:54.104 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate[71657]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T15:35:54.104 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate[71657]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T15:35:54.104 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate[71657]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T15:35:54.104 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate[71657]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T15:35:54.104 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:54 smithi112 podman[71643]: 2024-04-26 15:35:54.056805948 +0000 UTC m=+1.183494997 container died 5f4999ed9af80c236dde75e3f13829d2a75705b74a93157713067386bc0484c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=) 2024-04-26T15:35:54.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 199173 11.09 MB/sec execute 318 sec latency 62.936 ms 2024-04-26T15:35:55.117 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:55 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:55.117 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:55 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:35:55.117 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:55 smithi112 ceph-mon[63158]: pgmap v98: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 352 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.7 MiB/s wr, 592 op/s; 196/1683 objects degraded (11.646%) 2024-04-26T15:35:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:55 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:55 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:35:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:55 smithi164 ceph-mon[49952]: pgmap v98: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 352 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.7 MiB/s wr, 592 op/s; 196/1683 objects degraded (11.646%) 2024-04-26T15:35:55.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 199416 11.08 MB/sec execute 319 sec latency 59.679 ms 2024-04-26T15:35:55.905 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:55 smithi112 podman[71800]: 2024-04-26 15:35:55.769324919 +0000 UTC m=+1.706107691 container remove 5f4999ed9af80c236dde75e3f13829d2a75705b74a93157713067386bc0484c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-26T15:35:56.159 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:55 smithi112 podman[71860]: 2024-04-26 15:35:55.900595049 +0000 UTC m=+0.022429193 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:35:56.159 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:56 smithi112 podman[71860]: 2024-04-26 15:35:56.101017971 +0000 UTC m=+0.222852099 container create d2979058d0eb20ffc25625b25a6441cd00843f7ad4679c40c0c1faad60a23526 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-26T15:35:56.678 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:56 smithi112 podman[71860]: 2024-04-26 15:35:56.436072912 +0000 UTC m=+0.557907039 container init d2979058d0eb20ffc25625b25a6441cd00843f7ad4679c40c0c1faad60a23526 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD) 2024-04-26T15:35:56.678 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:56 smithi112 podman[71860]: 2024-04-26 15:35:56.445238864 +0000 UTC m=+0.567072995 container start d2979058d0eb20ffc25625b25a6441cd00843f7ad4679c40c0c1faad60a23526 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-26T15:35:56.678 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:56 smithi112 ceph-osd[71878]: -- 172.21.15.112:0/3072405818 <== mon.0 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bc12ec3680 con 0x55bc12e8cc00 2024-04-26T15:35:56.678 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:56 smithi112 bash[71860]: d2979058d0eb20ffc25625b25a6441cd00843f7ad4679c40c0c1faad60a23526 2024-04-26T15:35:56.678 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:56 smithi112 systemd[1]: Started Ceph osd.1 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:35:56.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 199677 11.07 MB/sec execute 320 sec latency 76.438 ms 2024-04-26T15:35:57.192 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:35:57 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[71874]: 2024-04-26T15:35:57.034+0000 7f13bb0417c0 -1 Falling back to public interface 2024-04-26T15:35:57.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:57 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:57.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:57 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:57.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:57 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:35:57.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:57 smithi112 ceph-mon[63158]: pgmap v99: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.1 MiB/s wr, 438 op/s; 192/1686 objects degraded (11.388%) 2024-04-26T15:35:57.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 199938 11.04 MB/sec execute 321 sec latency 44.817 ms 2024-04-26T15:35:58.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:57 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:58.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:57 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:35:58.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:57 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:35:58.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:57 smithi164 ceph-mon[49952]: pgmap v99: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.1 MiB/s wr, 438 op/s; 192/1686 objects degraded (11.388%) 2024-04-26T15:35:58.765 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:58 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 192/1686 objects degraded (11.388%), 24 pgs degraded (PG_DEGRADED) 2024-04-26T15:35:58.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 200219 11.03 MB/sec execute 322 sec latency 67.583 ms 2024-04-26T15:35:59.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:58 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 192/1686 objects degraded (11.388%), 24 pgs degraded (PG_DEGRADED) 2024-04-26T15:35:59.793 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:35:59 smithi112 ceph-mon[63158]: pgmap v100: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.6 MiB/s wr, 394 op/s; 187/1653 objects degraded (11.313%) 2024-04-26T15:35:59.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 200976 11.06 MB/sec execute 323 sec latency 19.727 ms 2024-04-26T15:36:00.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:35:59 smithi164 ceph-mon[49952]: pgmap v100: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.6 MiB/s wr, 394 op/s; 187/1653 objects degraded (11.313%) 2024-04-26T15:36:00.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 201675 11.09 MB/sec execute 324 sec latency 77.134 ms 2024-04-26T15:36:01.136 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:01 smithi112 ceph-mon[63158]: pgmap v101: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 349 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 553 op/s; 198/1755 objects degraded (11.282%) 2024-04-26T15:36:01.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:01 smithi164 ceph-mon[49952]: pgmap v101: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 349 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 553 op/s; 198/1755 objects degraded (11.282%) 2024-04-26T15:36:01.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 202323 11.11 MB/sec execute 325 sec latency 23.853 ms 2024-04-26T15:36:02.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 202852 11.12 MB/sec execute 326 sec latency 34.773 ms 2024-04-26T15:36:03.155 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:03 smithi112 ceph-mon[63158]: pgmap v102: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 352 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.9 MiB/s wr, 460 op/s; 205/1803 objects degraded (11.370%) 2024-04-26T15:36:03.155 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:03 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 205/1803 objects degraded (11.370%), 24 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:03.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:03 smithi164 ceph-mon[49952]: pgmap v102: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 352 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.9 MiB/s wr, 460 op/s; 205/1803 objects degraded (11.370%) 2024-04-26T15:36:03.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:03 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 205/1803 objects degraded (11.370%), 24 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:03.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 203216 11.12 MB/sec execute 327 sec latency 51.040 ms 2024-04-26T15:36:04.162 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:36:03 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[71874]: 2024-04-26T15:36:03.961+0000 7f13bb0417c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-26T15:36:04.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 203527 11.10 MB/sec execute 328 sec latency 45.617 ms 2024-04-26T15:36:05.177 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:05 smithi112 ceph-mon[63158]: pgmap v103: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 6.6 MiB/s wr, 702 op/s; 194/1707 objects degraded (11.365%) 2024-04-26T15:36:05.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:05 smithi164 ceph-mon[49952]: pgmap v103: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 6.6 MiB/s wr, 702 op/s; 194/1707 objects degraded (11.365%) 2024-04-26T15:36:05.431 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:36:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[71874]: 2024-04-26T15:36:05.263+0000 7f13bb0417c0 -1 osd.1 83 log_to_monitors true 2024-04-26T15:36:05.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 203856 11.09 MB/sec execute 329 sec latency 46.186 ms 2024-04-26T15:36:06.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:06 smithi112 ceph-mon[63158]: from='osd.1 [v2:172.21.15.112:6810/2054112678,v1:172.21.15.112:6811/2054112678]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T15:36:06.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:06 smithi164 ceph-mon[49952]: from='osd.1 [v2:172.21.15.112:6810/2054112678,v1:172.21.15.112:6811/2054112678]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T15:36:06.754 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:36:06 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[71874]: 2024-04-26T15:36:06.605+0000 7f13a8491700 -1 osd.1 83 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:36:06.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 204187 11.08 MB/sec execute 330 sec latency 141.079 ms 2024-04-26T15:36:07.258 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:07 smithi112 ceph-mon[63158]: from='osd.1 [v2:172.21.15.112:6810/2054112678,v1:172.21.15.112:6811/2054112678]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T15:36:07.259 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:07 smithi112 ceph-mon[63158]: osdmap e86: 8 total, 7 up, 8 in 2024-04-26T15:36:07.259 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:07 smithi112 ceph-mon[63158]: from='osd.1 [v2:172.21.15.112:6810/2054112678,v1:172.21.15.112:6811/2054112678]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:36:07.259 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:07 smithi112 ceph-mon[63158]: from='osd.1 [v2:172.21.15.112:6810/2054112678,v1:172.21.15.112:6811/2054112678]' entity='osd.1' 2024-04-26T15:36:07.259 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:07 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:07.259 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:07 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:07.259 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:07 smithi112 ceph-mon[63158]: pgmap v105: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.3 MiB/s wr, 638 op/s; 180/1668 objects degraded (10.791%) 2024-04-26T15:36:07.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:07 smithi164 ceph-mon[49952]: from='osd.1 [v2:172.21.15.112:6810/2054112678,v1:172.21.15.112:6811/2054112678]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T15:36:07.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:07 smithi164 ceph-mon[49952]: osdmap e86: 8 total, 7 up, 8 in 2024-04-26T15:36:07.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:07 smithi164 ceph-mon[49952]: from='osd.1 [v2:172.21.15.112:6810/2054112678,v1:172.21.15.112:6811/2054112678]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:36:07.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:07 smithi164 ceph-mon[49952]: from='osd.1 [v2:172.21.15.112:6810/2054112678,v1:172.21.15.112:6811/2054112678]' entity='osd.1' 2024-04-26T15:36:07.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:07 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:07.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:07 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:07.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:07 smithi164 ceph-mon[49952]: pgmap v105: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.3 MiB/s wr, 638 op/s; 180/1668 objects degraded (10.791%) 2024-04-26T15:36:07.865 INFO:teuthology.orchestra.run.smithi112.stdout: 5 204467 11.07 MB/sec execute 331 sec latency 268.344 ms 2024-04-26T15:36:08.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:08 smithi112 ceph-mon[63158]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:36:08.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:08 smithi112 ceph-mon[63158]: osd.1 [v2:172.21.15.112:6810/2054112678,v1:172.21.15.112:6811/2054112678] boot 2024-04-26T15:36:08.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:08 smithi112 ceph-mon[63158]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T15:36:08.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:36:08.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:08.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:08.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:08 smithi164 ceph-mon[49952]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:36:08.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:08 smithi164 ceph-mon[49952]: osd.1 [v2:172.21.15.112:6810/2054112678,v1:172.21.15.112:6811/2054112678] boot 2024-04-26T15:36:08.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:08 smithi164 ceph-mon[49952]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T15:36:08.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:36:08.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:08.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:08.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 204479 11.04 MB/sec execute 332 sec latency 1268.374 ms 2024-04-26T15:36:09.716 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:09 smithi112 ceph-mon[63158]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T15:36:09.716 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:09 smithi112 ceph-mon[63158]: pgmap v108: 97 pgs: 3 peering, 23 active+undersized+degraded, 13 active+undersized, 58 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 594 op/s; 161/1632 objects degraded (9.865%) 2024-04-26T15:36:09.716 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:09.717 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:36:09.717 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:09 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 180/1668 objects degraded (10.791%), 24 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:09.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:09 smithi164 ceph-mon[49952]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T15:36:09.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:09 smithi164 ceph-mon[49952]: pgmap v108: 97 pgs: 3 peering, 23 active+undersized+degraded, 13 active+undersized, 58 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 594 op/s; 161/1632 objects degraded (9.865%) 2024-04-26T15:36:09.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:09.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:36:09.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:09 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 180/1668 objects degraded (10.791%), 24 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:09.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 204577 11.03 MB/sec execute 333 sec latency 2080.259 ms 2024-04-26T15:36:10.727 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:10 smithi112 ceph-mon[63158]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T15:36:10.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 204986 11.03 MB/sec execute 334 sec latency 44.680 ms 2024-04-26T15:36:11.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:10 smithi164 ceph-mon[49952]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T15:36:11.741 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:11 smithi112 ceph-mon[63158]: pgmap v110: 97 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 3 peering, 6 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 72 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.5 MiB/s wr, 340 op/s; 208/1542 objects degraded (13.489%); 830 KiB/s, 6 objects/s recovering 2024-04-26T15:36:11.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 205319 11.03 MB/sec execute 335 sec latency 151.436 ms 2024-04-26T15:36:12.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:11 smithi164 ceph-mon[49952]: pgmap v110: 97 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 3 peering, 6 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 72 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.5 MiB/s wr, 340 op/s; 208/1542 objects degraded (13.489%); 830 KiB/s, 6 objects/s recovering 2024-04-26T15:36:12.755 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:12 smithi112 ceph-mon[63158]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T15:36:12.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 205636 11.01 MB/sec execute 336 sec latency 942.125 ms 2024-04-26T15:36:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:12 smithi164 ceph-mon[49952]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T15:36:13.774 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:13 smithi112 ceph-mon[63158]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T15:36:13.775 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:13 smithi112 ceph-mon[63158]: pgmap v113: 97 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.9 MiB/s wr, 364 op/s; 186/1536 objects degraded (12.109%); 1.0 MiB/s, 83 keys/s, 13 objects/s recovering 2024-04-26T15:36:13.775 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:13.775 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:13.775 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:36:13.775 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:36:13.775 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:13.775 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:13.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 205800 10.99 MB/sec execute 337 sec latency 951.078 ms 2024-04-26T15:36:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:13 smithi164 ceph-mon[49952]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T15:36:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:13 smithi164 ceph-mon[49952]: pgmap v113: 97 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.9 MiB/s wr, 364 op/s; 186/1536 objects degraded (12.109%); 1.0 MiB/s, 83 keys/s, 13 objects/s recovering 2024-04-26T15:36:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:14.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:36:14.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:36:14.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:14.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:14.807 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:14 smithi112 ceph-mon[63158]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T15:36:14.807 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:36:14.807 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:36:14.807 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:36:14.807 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:36:14.807 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T15:36:14.807 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:14 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T15:36:14.807 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:14 smithi112 ceph-mon[63158]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T15:36:14.807 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:14 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 186/1536 objects degraded (12.109%), 11 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:14.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 206128 10.98 MB/sec execute 338 sec latency 957.156 ms 2024-04-26T15:36:15.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:14 smithi164 ceph-mon[49952]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T15:36:15.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:36:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:36:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:36:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:36:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T15:36:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:14 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T15:36:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:14 smithi164 ceph-mon[49952]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T15:36:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:14 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 186/1536 objects degraded (12.109%), 11 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:15.816 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:15 smithi112 ceph-mon[63158]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T15:36:15.816 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:15 smithi112 ceph-mon[63158]: pgmap v116: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 267 op/s; 75/1587 objects degraded (4.726%); 1.1 MiB/s, 92 keys/s, 34 objects/s recovering 2024-04-26T15:36:15.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 206521 10.98 MB/sec execute 339 sec latency 39.381 ms 2024-04-26T15:36:16.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:15 smithi164 ceph-mon[49952]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T15:36:16.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:15 smithi164 ceph-mon[49952]: pgmap v116: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 267 op/s; 75/1587 objects degraded (4.726%); 1.1 MiB/s, 92 keys/s, 34 objects/s recovering 2024-04-26T15:36:16.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 206886 10.97 MB/sec execute 340 sec latency 52.230 ms 2024-04-26T15:36:17.355 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:17 smithi112 ceph-mon[63158]: pgmap v117: 97 pgs: 1 peering, 96 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 417 op/s; 851 KiB/s, 39 objects/s recovering 2024-04-26T15:36:17.356 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:17 smithi112 ceph-mon[63158]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 75/1587 objects degraded (4.726%), 4 pgs degraded) 2024-04-26T15:36:17.356 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:17 smithi112 ceph-mon[63158]: Cluster is now healthy 2024-04-26T15:36:17.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:17 smithi164 ceph-mon[49952]: pgmap v117: 97 pgs: 1 peering, 96 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 417 op/s; 851 KiB/s, 39 objects/s recovering 2024-04-26T15:36:17.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:17 smithi164 ceph-mon[49952]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 75/1587 objects degraded (4.726%), 4 pgs degraded) 2024-04-26T15:36:17.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:17 smithi164 ceph-mon[49952]: Cluster is now healthy 2024-04-26T15:36:17.634 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:36:17.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 207218 10.96 MB/sec execute 341 sec latency 55.667 ms 2024-04-26T15:36:18.118 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:18 smithi112 ceph-mon[63158]: from='client.34252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:18.118 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:18 smithi112 ceph-mon[63158]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:18.560 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:36:18.560 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (5m) 11s ago 14m 24.6M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (112s) 11s ago 14m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (105s) 101s ago 13m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (5m) 11s ago 14m 54.4M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (8m) 11s ago 8m 38.0M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (8m) 101s ago 8m 14.0M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (4m) 11s ago 15m 518M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (3m) 101s ago 13m 465M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (3m) 11s ago 15m 48.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (2m) 101s ago 13m 35.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (7m) 11s ago 7m 262M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (7m) 101s ago 7m 87.6M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (4m) 11s ago 13m 20.3M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (4m) 101s ago 13m 17.6M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (71s) 11s ago 12m 255M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:36:18.561 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (22s) 11s ago 12m 12.1M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:36:18.562 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (11m) 11s ago 11m 515M 1936M 17.2.0 e1d6a67b021e 7ba4905e502c 2024-04-26T15:36:18.562 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (11m) 11s ago 11m 377M 1936M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:36:18.562 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (10m) 101s ago 10m 551M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:36:18.562 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (10m) 101s ago 10m 373M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:36:18.562 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (10m) 101s ago 10m 557M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:36:18.562 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (9m) 101s ago 9m 448M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:36:18.562 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (3m) 11s ago 13m 72.1M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:36:18.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:18 smithi164 ceph-mon[49952]: from='client.34252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:18.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:18 smithi164 ceph-mon[49952]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:18.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 207462 10.96 MB/sec execute 342 sec latency 58.011 ms 2024-04-26T15:36:19.031 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:36:19.031 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:36:19.031 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:36:19.031 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:36:19.031 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:36:19.031 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:36:19.031 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:36:19.031 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:36:19.032 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-26T15:36:19.032 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:36:19.032 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:36:19.032 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:36:19.032 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:36:19.032 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:36:19.032 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:36:19.032 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-26T15:36:19.032 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 6 2024-04-26T15:36:19.032 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:36:19.032 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:36:19.161 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:19 smithi112 ceph-mon[63158]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:19.161 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:19 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/1378212883' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:36:19.161 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:19 smithi112 ceph-mon[63158]: pgmap v118: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.8 MiB/s wr, 379 op/s; 720 KiB/s, 33 objects/s recovering 2024-04-26T15:36:19.450 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:36:19.451 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:36:19.451 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:36:19.451 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:36:19.451 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:36:19.451 INFO:teuthology.orchestra.run.smithi112.stdout: "crash", 2024-04-26T15:36:19.451 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:36:19.451 INFO:teuthology.orchestra.run.smithi112.stdout: "mon" 2024-04-26T15:36:19.451 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:36:19.451 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "8/23 daemons upgraded", 2024-04-26T15:36:19.451 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T15:36:19.451 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:36:19.451 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:36:19.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:19 smithi164 ceph-mon[49952]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:19.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:19 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/1378212883' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:36:19.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:19 smithi164 ceph-mon[49952]: pgmap v118: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.8 MiB/s wr, 379 op/s; 720 KiB/s, 33 objects/s recovering 2024-04-26T15:36:19.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 207707 10.95 MB/sec execute 343 sec latency 61.157 ms 2024-04-26T15:36:20.142 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:36:20.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:20 smithi112 ceph-mon[63158]: from='client.34268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:20.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:20 smithi164 ceph-mon[49952]: from='client.34268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:20.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 207985 10.95 MB/sec execute 344 sec latency 56.104 ms 2024-04-26T15:36:21.238 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:21 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/1132459355' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:36:21.238 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:21 smithi112 ceph-mon[63158]: pgmap v119: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 536 op/s; 512 KiB/s, 24 objects/s recovering 2024-04-26T15:36:21.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:21 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/1132459355' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:36:21.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:21 smithi164 ceph-mon[49952]: pgmap v119: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 536 op/s; 512 KiB/s, 24 objects/s recovering 2024-04-26T15:36:21.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 208226 10.94 MB/sec execute 345 sec latency 41.976 ms 2024-04-26T15:36:22.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 208550 10.93 MB/sec execute 346 sec latency 56.310 ms 2024-04-26T15:36:23.303 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:23 smithi112 ceph-mon[63158]: pgmap v120: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.9 MiB/s wr, 482 op/s; 435 KiB/s, 20 objects/s recovering 2024-04-26T15:36:23.399 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:23 smithi164 ceph-mon[49952]: pgmap v120: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.9 MiB/s wr, 482 op/s; 435 KiB/s, 20 objects/s recovering 2024-04-26T15:36:23.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 208804 10.91 MB/sec execute 347 sec latency 47.545 ms 2024-04-26T15:36:24.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 209018 10.89 MB/sec execute 348 sec latency 74.066 ms 2024-04-26T15:36:25.092 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:25 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:25.092 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:25 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:36:25.093 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:25 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:25.093 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:25 smithi112 ceph-mon[63158]: pgmap v121: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.3 MiB/s wr, 508 op/s; 393 KiB/s, 18 objects/s recovering 2024-04-26T15:36:25.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:25 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:25.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:25 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:36:25.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:25 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:25.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:25 smithi164 ceph-mon[49952]: pgmap v121: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.3 MiB/s wr, 508 op/s; 393 KiB/s, 18 objects/s recovering 2024-04-26T15:36:25.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 209364 10.89 MB/sec execute 349 sec latency 49.354 ms 2024-04-26T15:36:26.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 209743 10.88 MB/sec execute 350 sec latency 41.656 ms 2024-04-26T15:36:27.104 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:27 smithi112 ceph-mon[63158]: pgmap v122: 97 pgs: 97 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.5 MiB/s wr, 459 op/s; 0 B/s, 6 objects/s recovering 2024-04-26T15:36:27.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:27 smithi164 ceph-mon[49952]: pgmap v122: 97 pgs: 97 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.5 MiB/s wr, 459 op/s; 0 B/s, 6 objects/s recovering 2024-04-26T15:36:27.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 210132 10.88 MB/sec execute 351 sec latency 43.429 ms 2024-04-26T15:36:28.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 210475 10.87 MB/sec execute 352 sec latency 48.342 ms 2024-04-26T15:36:28.883 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T15:36:29.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T15:36:29.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 210827 10.86 MB/sec execute 353 sec latency 44.817 ms 2024-04-26T15:36:29.891 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:29 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T15:36:29.891 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:29 smithi112 ceph-mon[63158]: Upgrade: osd.2 is safe to restart 2024-04-26T15:36:29.891 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:29 smithi112 ceph-mon[63158]: pgmap v123: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 371 op/s 2024-04-26T15:36:30.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:29 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T15:36:30.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:29 smithi164 ceph-mon[49952]: Upgrade: osd.2 is safe to restart 2024-04-26T15:36:30.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:29 smithi164 ceph-mon[49952]: pgmap v123: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 371 op/s 2024-04-26T15:36:30.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 211092 10.86 MB/sec execute 354 sec latency 37.102 ms 2024-04-26T15:36:31.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 211346 10.86 MB/sec execute 355 sec latency 57.647 ms 2024-04-26T15:36:32.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:32 smithi112 ceph-mon[63158]: pgmap v124: 97 pgs: 97 active+clean; 353 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 542 op/s 2024-04-26T15:36:32.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:32 smithi112 ceph-mon[63158]: Upgrade: Updating osd.2 2024-04-26T15:36:32.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:32 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:32.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:32 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T15:36:32.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:32 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:36:32.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:32 smithi112 ceph-mon[63158]: Deploying daemon osd.2 on smithi112 2024-04-26T15:36:32.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:32 smithi164 ceph-mon[49952]: pgmap v124: 97 pgs: 97 active+clean; 353 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 542 op/s 2024-04-26T15:36:32.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:32 smithi164 ceph-mon[49952]: Upgrade: Updating osd.2 2024-04-26T15:36:32.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:32 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:32.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:32 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T15:36:32.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:32 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:36:32.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:32 smithi164 ceph-mon[49952]: Deploying daemon osd.2 on smithi112 2024-04-26T15:36:32.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 211651 10.86 MB/sec execute 356 sec latency 49.016 ms 2024-04-26T15:36:33.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 211987 10.85 MB/sec execute 357 sec latency 38.387 ms 2024-04-26T15:36:34.478 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:34 smithi112 ceph-mon[63158]: pgmap v125: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.4 MiB/s wr, 402 op/s 2024-04-26T15:36:34.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:34 smithi164 ceph-mon[49952]: pgmap v125: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.4 MiB/s wr, 402 op/s 2024-04-26T15:36:34.867 INFO:teuthology.orchestra.run.smithi112.stdout: 5 212317 10.84 MB/sec execute 358 sec latency 57.171 ms 2024-04-26T15:36:35.866 INFO:teuthology.orchestra.run.smithi112.stdout: 5 212672 10.83 MB/sec execute 359 sec latency 35.066 ms 2024-04-26T15:36:36.487 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:36 smithi112 ceph-mon[63158]: pgmap v126: 97 pgs: 97 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.1 MiB/s wr, 541 op/s 2024-04-26T15:36:36.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:36 smithi164 ceph-mon[49952]: pgmap v126: 97 pgs: 97 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.1 MiB/s wr, 541 op/s 2024-04-26T15:36:36.867 INFO:teuthology.orchestra.run.smithi112.stdout: 5 213122 10.84 MB/sec execute 360 sec latency 52.273 ms 2024-04-26T15:36:36.992 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:36 smithi112 systemd[1]: Stopping Ceph osd.2 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:36:37.495 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:37 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2[38985]: 2024-04-26T15:36:37.409+0000 7f66d43a9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:36:37.496 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:37 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2[38985]: 2024-04-26T15:36:37.409+0000 7f66d43a9700 -1 osd.2 93 *** Got signal Terminated *** 2024-04-26T15:36:37.496 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:37 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2[38985]: 2024-04-26T15:36:37.409+0000 7f66d43a9700 -1 osd.2 93 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:36:37.867 INFO:teuthology.orchestra.run.smithi112.stdout: 5 213594 10.84 MB/sec execute 361 sec latency 39.688 ms 2024-04-26T15:36:38.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:38 smithi112 ceph-mon[63158]: pgmap v127: 97 pgs: 97 active+clean; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 495 op/s 2024-04-26T15:36:38.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:38 smithi112 ceph-mon[63158]: osd.2 marked itself down and dead 2024-04-26T15:36:38.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:38 smithi164 ceph-mon[49952]: pgmap v127: 97 pgs: 97 active+clean; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 495 op/s 2024-04-26T15:36:38.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:38 smithi164 ceph-mon[49952]: osd.2 marked itself down and dead 2024-04-26T15:36:38.867 INFO:teuthology.orchestra.run.smithi112.stdout: 5 213855 10.82 MB/sec execute 362 sec latency 438.106 ms 2024-04-26T15:36:39.405 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:39 smithi112 podman[74030]: 2024-04-26 15:36:39.113010452 +0000 UTC m=+2.141312863 container died 7ba4905e502c5b71dd042dc6140f7f22c610711edd3639a5be9a20c472bcfdf6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, release=754, io.buildah.version=1.19.8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-26T15:36:39.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:39 smithi112 ceph-mon[63158]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:36:39.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:39 smithi112 ceph-mon[63158]: osdmap e94: 8 total, 7 up, 8 in 2024-04-26T15:36:39.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:39.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:36:39.657 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:39 smithi112 podman[74030]: 2024-04-26 15:36:39.618091409 +0000 UTC m=+2.646393814 container cleanup 7ba4905e502c5b71dd042dc6140f7f22c610711edd3639a5be9a20c472bcfdf6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.buildah.version=1.19.8, version=8, name=centos-stream, release=754, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_CLEAN=True, distribution-scope=public, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-26T15:36:39.657 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:39 smithi112 bash[74030]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2 2024-04-26T15:36:39.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:39 smithi164 ceph-mon[49952]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:36:39.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:39 smithi164 ceph-mon[49952]: osdmap e94: 8 total, 7 up, 8 in 2024-04-26T15:36:39.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:39.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:36:39.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 214077 10.81 MB/sec execute 363 sec latency 1036.263 ms 2024-04-26T15:36:40.162 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:40 smithi112 podman[74052]: 2024-04-26 15:36:40.005920032 +0000 UTC m=+0.897293926 container remove 7ba4905e502c5b71dd042dc6140f7f22c610711edd3639a5be9a20c472bcfdf6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-type=git, version=8, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, ceph=True) 2024-04-26T15:36:40.677 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:40 smithi112 podman[74132]: 2024-04-26 15:36:40.507125652 +0000 UTC m=+0.029862486 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:36:40.678 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:40 smithi112 ceph-mon[63158]: pgmap v129: 97 pgs: 6 stale+active+clean, 91 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 477 op/s 2024-04-26T15:36:40.678 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:40 smithi112 ceph-mon[63158]: osdmap e95: 8 total, 7 up, 8 in 2024-04-26T15:36:40.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:40 smithi164 ceph-mon[49952]: pgmap v129: 97 pgs: 6 stale+active+clean, 91 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 477 op/s 2024-04-26T15:36:40.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:40 smithi164 ceph-mon[49952]: osdmap e95: 8 total, 7 up, 8 in 2024-04-26T15:36:40.867 INFO:teuthology.orchestra.run.smithi112.stdout: 5 214543 10.81 MB/sec execute 364 sec latency 30.235 ms 2024-04-26T15:36:40.928 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:40 smithi112 podman[74132]: 2024-04-26 15:36:40.698111082 +0000 UTC m=+0.220847885 container create af89160c7028d1ad28f06b55ae26407c9aeacde76c7f8afcc69ab060e3c13595 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-deactivate, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-26T15:36:41.179 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:41 smithi112 podman[74132]: 2024-04-26 15:36:41.099647051 +0000 UTC m=+0.622383863 container init af89160c7028d1ad28f06b55ae26407c9aeacde76c7f8afcc69ab060e3c13595 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:36:41.180 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:41 smithi112 podman[74132]: 2024-04-26 15:36:41.109766763 +0000 UTC m=+0.632503571 container start af89160c7028d1ad28f06b55ae26407c9aeacde76c7f8afcc69ab060e3c13595 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-26T15:36:41.436 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:41 smithi112 podman[74132]: 2024-04-26 15:36:41.274557116 +0000 UTC m=+0.797293923 container attach af89160c7028d1ad28f06b55ae26407c9aeacde76c7f8afcc69ab060e3c13595 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-26T15:36:41.688 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:41 smithi112 ceph-mon[63158]: Health check failed: Degraded data redundancy: 105/1641 objects degraded (6.399%), 14 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:41.689 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:41 smithi112 conmon[74147]: conmon af89160c7028d1ad28f0 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-af89160c7028d1ad28f06b55ae26407c9aeacde76c7f8afcc69ab060e3c13595.scope/container/memory.events 2024-04-26T15:36:41.689 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:41 smithi112 podman[74132]: 2024-04-26 15:36:41.570424056 +0000 UTC m=+1.093160876 container died af89160c7028d1ad28f06b55ae26407c9aeacde76c7f8afcc69ab060e3c13595 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-26T15:36:41.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:41 smithi164 ceph-mon[49952]: Health check failed: Degraded data redundancy: 105/1641 objects degraded (6.399%), 14 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:41.867 INFO:teuthology.orchestra.run.smithi112.stdout: 5 214959 10.83 MB/sec execute 365 sec latency 33.497 ms 2024-04-26T15:36:42.698 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:42 smithi112 ceph-mon[63158]: pgmap v131: 97 pgs: 14 active+undersized+degraded, 3 stale+active+clean, 3 active+undersized, 77 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.9 MiB/s wr, 533 op/s; 105/1641 objects degraded (6.399%) 2024-04-26T15:36:42.765 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:42 smithi112 podman[74158]: 2024-04-26 15:36:42.714218021 +0000 UTC m=+1.134719265 container remove af89160c7028d1ad28f06b55ae26407c9aeacde76c7f8afcc69ab060e3c13595 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD) 2024-04-26T15:36:42.765 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:42 smithi112 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-26T15:36:42.765 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:42 smithi112 systemd[1]: Stopped Ceph osd.2 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:36:42.765 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:42 smithi112 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.2.service: Consumed 3min 935ms CPU time. 2024-04-26T15:36:42.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:42 smithi164 ceph-mon[49952]: pgmap v131: 97 pgs: 14 active+undersized+degraded, 3 stale+active+clean, 3 active+undersized, 77 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.9 MiB/s wr, 533 op/s; 105/1641 objects degraded (6.399%) 2024-04-26T15:36:42.867 INFO:teuthology.orchestra.run.smithi112.stdout: 5 215399 10.84 MB/sec execute 366 sec latency 35.899 ms 2024-04-26T15:36:43.209 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:43 smithi112 systemd[1]: Starting Ceph osd.2 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:36:43.461 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:43 smithi112 podman[74283]: 2024-04-26 15:36:43.249567189 +0000 UTC m=+0.029840948 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:36:43.461 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:43 smithi112 podman[74283]: 2024-04-26 15:36:43.3994765 +0000 UTC m=+0.179750256 container create ae947b0defac2d8e4f1bca6e56f0e2b0381b9a7de6f0d3cec9bd585ddb02d19a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=) 2024-04-26T15:36:43.867 INFO:teuthology.orchestra.run.smithi112.stdout: 5 215801 10.83 MB/sec execute 367 sec latency 42.719 ms 2024-04-26T15:36:43.967 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:43 smithi112 podman[74283]: 2024-04-26 15:36:43.943488783 +0000 UTC m=+0.723762548 container init ae947b0defac2d8e4f1bca6e56f0e2b0381b9a7de6f0d3cec9bd585ddb02d19a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T15:36:43.968 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:43 smithi112 podman[74283]: 2024-04-26 15:36:43.949649245 +0000 UTC m=+0.729923002 container start ae947b0defac2d8e4f1bca6e56f0e2b0381b9a7de6f0d3cec9bd585ddb02d19a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T15:36:44.470 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 podman[74283]: 2024-04-26 15:36:44.38799808 +0000 UTC m=+1.168271838 container attach ae947b0defac2d8e4f1bca6e56f0e2b0381b9a7de6f0d3cec9bd585ddb02d19a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:36:44.732 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:44 smithi112 ceph-mon[63158]: pgmap v132: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.6 MiB/s wr, 439 op/s; 157/1614 objects degraded (9.727%) 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate[74298]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate[74298]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 bash[74283]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 bash[74283]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 bash[74283]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 bash[74283]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 bash[74283]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 bash[74283]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 bash[74283]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 bash[74283]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 bash[74283]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate[74298]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T15:36:44.733 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate[74298]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T15:36:44.734 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate[74298]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T15:36:44.734 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate[74298]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T15:36:44.734 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate[74298]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T15:36:44.734 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate[74298]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T15:36:44.734 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate[74298]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T15:36:44.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:44 smithi164 ceph-mon[49952]: pgmap v132: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.6 MiB/s wr, 439 op/s; 157/1614 objects degraded (9.727%) 2024-04-26T15:36:44.867 INFO:teuthology.orchestra.run.smithi112.stdout: 5 216041 10.82 MB/sec execute 368 sec latency 42.703 ms 2024-04-26T15:36:44.983 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:44 smithi112 podman[74283]: 2024-04-26 15:36:44.736001035 +0000 UTC m=+1.516274787 container died ae947b0defac2d8e4f1bca6e56f0e2b0381b9a7de6f0d3cec9bd585ddb02d19a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T15:36:45.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 216546 10.82 MB/sec execute 369 sec latency 35.042 ms 2024-04-26T15:36:46.020 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:45 smithi112 podman[74440]: 2024-04-26 15:36:45.825291979 +0000 UTC m=+1.084028844 container remove ae947b0defac2d8e4f1bca6e56f0e2b0381b9a7de6f0d3cec9bd585ddb02d19a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:36:46.274 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:46 smithi112 podman[74501]: 2024-04-26 15:36:45.983029187 +0000 UTC m=+0.023873901 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:36:46.274 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:46 smithi112 podman[74501]: 2024-04-26 15:36:46.132674927 +0000 UTC m=+0.173519631 container create a6c099476ffcc9c846927963e9943af17d937a9375d0469e166cb3c340a5b9a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-26T15:36:46.526 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:46 smithi112 ceph-mon[63158]: pgmap v133: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.0 MiB/s wr, 625 op/s; 160/1701 objects degraded (9.406%) 2024-04-26T15:36:46.527 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:46 smithi112 podman[74501]: 2024-04-26 15:36:46.478543068 +0000 UTC m=+0.519387778 container init a6c099476ffcc9c846927963e9943af17d937a9375d0469e166cb3c340a5b9a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:36:46.527 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:46 smithi112 podman[74501]: 2024-04-26 15:36:46.489105076 +0000 UTC m=+0.529949780 container start a6c099476ffcc9c846927963e9943af17d937a9375d0469e166cb3c340a5b9a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-26T15:36:46.527 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:46 smithi112 ceph-osd[74519]: -- 172.21.15.112:0/1751001964 <== mon.0 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5582acb4b680 con 0x5582acb15000 2024-04-26T15:36:46.782 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:46 smithi112 bash[74501]: a6c099476ffcc9c846927963e9943af17d937a9375d0469e166cb3c340a5b9a3 2024-04-26T15:36:46.782 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:46 smithi112 systemd[1]: Started Ceph osd.2 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:36:46.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:46 smithi164 ceph-mon[49952]: pgmap v133: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.0 MiB/s wr, 625 op/s; 160/1701 objects degraded (9.406%) 2024-04-26T15:36:46.867 INFO:teuthology.orchestra.run.smithi112.stdout: 5 217009 10.82 MB/sec execute 370 sec latency 35.379 ms 2024-04-26T15:36:47.286 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:47 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2[74515]: 2024-04-26T15:36:47.068+0000 7f90dcc6c7c0 -1 Falling back to public interface 2024-04-26T15:36:47.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:47 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:47.790 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:47 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:47.790 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:47 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:36:47.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 217556 10.83 MB/sec execute 371 sec latency 32.469 ms 2024-04-26T15:36:48.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:47 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:48.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:47 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:48.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:47 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:36:48.793 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:48 smithi112 ceph-mon[63158]: pgmap v134: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.3 MiB/s wr, 622 op/s; 162/1722 objects degraded (9.408%) 2024-04-26T15:36:48.794 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:48 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 162/1722 objects degraded (9.408%), 19 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:48.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 218284 10.85 MB/sec execute 372 sec latency 46.156 ms 2024-04-26T15:36:49.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:48 smithi164 ceph-mon[49952]: pgmap v134: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.3 MiB/s wr, 622 op/s; 162/1722 objects degraded (9.408%) 2024-04-26T15:36:49.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:48 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 162/1722 objects degraded (9.408%), 19 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:49.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 218736 10.88 MB/sec execute 373 sec latency 52.302 ms 2024-04-26T15:36:50.687 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:36:50.847 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:50 smithi112 ceph-mon[63158]: pgmap v135: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.7 MiB/s wr, 555 op/s; 165/1728 objects degraded (9.549%) 2024-04-26T15:36:50.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 219036 10.87 MB/sec execute 374 sec latency 46.406 ms 2024-04-26T15:36:51.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:50 smithi164 ceph-mon[49952]: pgmap v135: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.7 MiB/s wr, 555 op/s; 165/1728 objects degraded (9.549%) 2024-04-26T15:36:51.687 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:36:51.687 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (5m) 45s ago 14m 24.6M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:36:51.687 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (2m) 45s ago 14m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:36:51.687 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (2m) 2m ago 14m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:36:51.687 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (5m) 45s ago 14m 54.4M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:36:51.687 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (8m) 45s ago 8m 38.0M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:36:51.687 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (8m) 2m ago 8m 14.0M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (5m) 45s ago 16m 518M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (4m) 2m ago 14m 465M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (3m) 45s ago 16m 48.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (3m) 2m ago 14m 35.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (8m) 45s ago 8m 262M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (8m) 2m ago 8m 87.6M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (5m) 45s ago 14m 20.3M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (5m) 2m ago 13m 17.6M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (104s) 45s ago 13m 255M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (55s) 45s ago 12m 12.1M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 starting - - - 1936M 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (11m) 45s ago 11m 377M 1936M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (11m) 2m ago 11m 551M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (11m) 2m ago 11m 373M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (10m) 2m ago 10m 557M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:36:51.688 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (10m) 2m ago 10m 448M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:36:51.689 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (4m) 45s ago 13m 72.1M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:36:51.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 219369 10.86 MB/sec execute 375 sec latency 50.725 ms 2024-04-26T15:36:51.890 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:51 smithi112 ceph-mon[63158]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:52.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:51 smithi164 ceph-mon[49952]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:52.418 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:36:52.418 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:36:52.418 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:36:52.418 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 6 2024-04-26T15:36:52.419 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:36:52.420 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:36:52.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 219553 10.84 MB/sec execute 376 sec latency 61.584 ms 2024-04-26T15:36:52.915 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:52 smithi112 ceph-mon[63158]: pgmap v136: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 354 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 765 op/s; 178/1776 objects degraded (10.023%) 2024-04-26T15:36:52.915 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:52 smithi112 ceph-mon[63158]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:52.915 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:52 smithi112 ceph-mon[63158]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:52.915 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:52 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/3056213309' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:36:52.998 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:52 smithi164 ceph-mon[49952]: pgmap v136: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 354 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 765 op/s; 178/1776 objects degraded (10.023%) 2024-04-26T15:36:52.998 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:52 smithi164 ceph-mon[49952]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:52.998 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:52 smithi164 ceph-mon[49952]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:52.998 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:52 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/3056213309' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:36:53.049 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:36:53.049 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:36:53.050 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:36:53.050 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:36:53.050 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:36:53.050 INFO:teuthology.orchestra.run.smithi112.stdout: "crash", 2024-04-26T15:36:53.050 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:36:53.050 INFO:teuthology.orchestra.run.smithi112.stdout: "mon" 2024-04-26T15:36:53.050 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:36:53.050 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "8/23 daemons upgraded", 2024-04-26T15:36:53.050 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T15:36:53.050 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:36:53.050 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:36:53.422 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:53 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2[74515]: 2024-04-26T15:36:53.322+0000 7f90dcc6c7c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-26T15:36:53.735 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 179/1767 objects degraded (10.130%), 19 pgs degraded 2024-04-26T15:36:53.735 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-26T15:36:53.735 INFO:teuthology.orchestra.run.smithi112.stdout: osd.2 (root=default,host=smithi112) is down 2024-04-26T15:36:53.735 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 179/1767 objects degraded (10.130%), 19 pgs degraded 2024-04-26T15:36:53.735 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-04-26T15:36:53.735 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-04-26T15:36:53.736 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-04-26T15:36:53.737 INFO:teuthology.orchestra.run.smithi112.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-04-26T15:36:53.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 219801 10.83 MB/sec execute 377 sec latency 52.797 ms 2024-04-26T15:36:53.931 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:53 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 179/1767 objects degraded (10.130%), 19 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:53.931 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:53 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/3313293041' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:36:54.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:53 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 179/1767 objects degraded (10.130%), 19 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:54.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:53 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/3313293041' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:36:54.684 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:54 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2[74515]: 2024-04-26T15:36:54.533+0000 7f90dcc6c7c0 -1 osd.2 93 log_to_monitors true 2024-04-26T15:36:54.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 220092 10.82 MB/sec execute 378 sec latency 49.780 ms 2024-04-26T15:36:54.940 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:54 smithi112 ceph-mon[63158]: pgmap v137: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.5 MiB/s wr, 603 op/s; 179/1767 objects degraded (10.130%) 2024-04-26T15:36:54.940 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:54 smithi112 ceph-mon[63158]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:54.940 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:54.940 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:36:54.941 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:54 smithi112 ceph-mon[63158]: from='osd.2 [v2:172.21.15.112:6818/1419798793,v1:172.21.15.112:6819/1419798793]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T15:36:54.941 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:54 smithi112 ceph-mon[63158]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T15:36:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:54 smithi164 ceph-mon[49952]: pgmap v137: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.5 MiB/s wr, 603 op/s; 179/1767 objects degraded (10.130%) 2024-04-26T15:36:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:54 smithi164 ceph-mon[49952]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:36:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:36:55.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:54 smithi164 ceph-mon[49952]: from='osd.2 [v2:172.21.15.112:6818/1419798793,v1:172.21.15.112:6819/1419798793]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T15:36:55.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:54 smithi164 ceph-mon[49952]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T15:36:55.711 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:36:55 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2[74515]: 2024-04-26T15:36:55.628+0000 7f90ca0bc700 -1 osd.2 93 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:36:55.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 220325 10.81 MB/sec execute 379 sec latency 155.326 ms 2024-04-26T15:36:56.228 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:56 smithi112 ceph-mon[63158]: pgmap v138: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 659 op/s; 168/1656 objects degraded (10.145%) 2024-04-26T15:36:56.228 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:56 smithi112 ceph-mon[63158]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T15:36:56.228 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:56 smithi112 ceph-mon[63158]: osdmap e96: 8 total, 7 up, 8 in 2024-04-26T15:36:56.228 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:56 smithi112 ceph-mon[63158]: from='osd.2 [v2:172.21.15.112:6818/1419798793,v1:172.21.15.112:6819/1419798793]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:36:56.228 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:56 smithi112 ceph-mon[63158]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:36:56.228 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:56 smithi112 ceph-mon[63158]: from='osd.2 ' entity='osd.2' 2024-04-26T15:36:56.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:56 smithi164 ceph-mon[49952]: pgmap v138: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 659 op/s; 168/1656 objects degraded (10.145%) 2024-04-26T15:36:56.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:56 smithi164 ceph-mon[49952]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T15:36:56.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:56 smithi164 ceph-mon[49952]: osdmap e96: 8 total, 7 up, 8 in 2024-04-26T15:36:56.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:56 smithi164 ceph-mon[49952]: from='osd.2 [v2:172.21.15.112:6818/1419798793,v1:172.21.15.112:6819/1419798793]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:36:56.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:56 smithi164 ceph-mon[49952]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:36:56.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:56 smithi164 ceph-mon[49952]: from='osd.2 ' entity='osd.2' 2024-04-26T15:36:56.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 220606 10.79 MB/sec execute 380 sec latency 271.897 ms 2024-04-26T15:36:57.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:57 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:57.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:57 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:57.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:57 smithi112 ceph-mon[63158]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:36:57.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:57 smithi112 ceph-mon[63158]: osd.2 [v2:172.21.15.112:6818/1419798793,v1:172.21.15.112:6819/1419798793] boot 2024-04-26T15:36:57.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:57 smithi112 ceph-mon[63158]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T15:36:57.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:57 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:36:57.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:57 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:57.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:57 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:57.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:57 smithi164 ceph-mon[49952]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:36:57.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:57 smithi164 ceph-mon[49952]: osd.2 [v2:172.21.15.112:6818/1419798793,v1:172.21.15.112:6819/1419798793] boot 2024-04-26T15:36:57.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:57 smithi164 ceph-mon[49952]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T15:36:57.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:57 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:36:57.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 220606 10.76 MB/sec execute 381 sec latency 1271.930 ms 2024-04-26T15:36:58.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:58 smithi112 ceph-mon[63158]: pgmap v141: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 710 op/s; 162/1623 objects degraded (9.982%) 2024-04-26T15:36:58.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:58.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:58.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:58 smithi112 ceph-mon[63158]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T15:36:58.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:58 smithi164 ceph-mon[49952]: pgmap v141: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 710 op/s; 162/1623 objects degraded (9.982%) 2024-04-26T15:36:58.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:58.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:36:58.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:58 smithi164 ceph-mon[49952]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T15:36:58.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 220659 10.73 MB/sec execute 382 sec latency 2071.811 ms 2024-04-26T15:36:59.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 220808 10.72 MB/sec execute 383 sec latency 556.659 ms 2024-04-26T15:36:59.903 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:59 smithi112 ceph-mon[63158]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T15:36:59.903 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:59 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 162/1623 objects degraded (9.982%), 19 pgs degraded (PG_DEGRADED) 2024-04-26T15:36:59.904 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:36:59 smithi112 ceph-mon[63158]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T15:37:00.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:59 smithi164 ceph-mon[49952]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T15:37:00.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:59 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 162/1623 objects degraded (9.982%), 19 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:00.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:36:59 smithi164 ceph-mon[49952]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T15:37:00.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:00 smithi112 ceph-mon[63158]: pgmap v144: 97 pgs: 2 remapped+peering, 17 active+undersized+degraded, 6 active+undersized, 72 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s wr, 108 op/s; 139/1587 objects degraded (8.759%) 2024-04-26T15:37:00.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:00 smithi112 ceph-mon[63158]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T15:37:00.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 220891 10.70 MB/sec execute 384 sec latency 1041.986 ms 2024-04-26T15:37:01.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:00 smithi164 ceph-mon[49952]: pgmap v144: 97 pgs: 2 remapped+peering, 17 active+undersized+degraded, 6 active+undersized, 72 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s wr, 108 op/s; 139/1587 objects degraded (8.759%) 2024-04-26T15:37:01.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:00 smithi164 ceph-mon[49952]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T15:37:01.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 221128 10.69 MB/sec execute 385 sec latency 1011.152 ms 2024-04-26T15:37:02.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:02 smithi112 ceph-mon[63158]: pgmap v147: 97 pgs: 84 active+clean, 2 active+undersized, 4 peering, 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+undersized+remapped; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.0 MiB/s wr, 259 op/s; 76/1488 objects degraded (5.108%); 2.2 MiB/s, 21 objects/s recovering 2024-04-26T15:37:02.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:02 smithi112 ceph-mon[63158]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T15:37:02.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:02 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:02.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:02 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:02.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:02 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:37:02.561 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:02 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:37:02.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:02 smithi164 ceph-mon[49952]: pgmap v147: 97 pgs: 84 active+clean, 2 active+undersized, 4 peering, 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+undersized+remapped; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.0 MiB/s wr, 259 op/s; 76/1488 objects degraded (5.108%); 2.2 MiB/s, 21 objects/s recovering 2024-04-26T15:37:02.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:02 smithi164 ceph-mon[49952]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T15:37:02.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:02 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:02 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:02 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:37:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:02 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:37:02.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 221531 10.68 MB/sec execute 386 sec latency 51.121 ms 2024-04-26T15:37:03.605 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:03 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:03.606 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:03 smithi112 ceph-mon[63158]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T15:37:03.606 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:03 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:03.606 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:03 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:37:03.606 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:03 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:03.606 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:03 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:03.606 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:03 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:03.606 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:03 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T15:37:03.606 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:03 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T15:37:03.606 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:03 smithi112 ceph-mon[63158]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T15:37:03.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:03 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:03.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:03 smithi164 ceph-mon[49952]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T15:37:03.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:03 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:03.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:03 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:37:03.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:03 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:03.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:03 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:03.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:03 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:03.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:03 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T15:37:03.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:03 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T15:37:03.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:03 smithi164 ceph-mon[49952]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T15:37:03.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 221834 10.68 MB/sec execute 387 sec latency 49.794 ms 2024-04-26T15:37:04.637 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:04 smithi112 ceph-mon[63158]: pgmap v150: 97 pgs: 87 active+clean, 5 peering, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.5 MiB/s wr, 227 op/s; 54/1506 objects degraded (3.586%); 2.5 MiB/s, 39 objects/s recovering 2024-04-26T15:37:04.638 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:04 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 54/1506 objects degraded (3.586%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:04.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:04 smithi164 ceph-mon[49952]: pgmap v150: 97 pgs: 87 active+clean, 5 peering, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.5 MiB/s wr, 227 op/s; 54/1506 objects degraded (3.586%); 2.5 MiB/s, 39 objects/s recovering 2024-04-26T15:37:04.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:04 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 54/1506 objects degraded (3.586%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:04.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 222039 10.67 MB/sec execute 388 sec latency 138.273 ms 2024-04-26T15:37:05.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 222262 10.66 MB/sec execute 389 sec latency 77.245 ms 2024-04-26T15:37:06.441 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:06 smithi112 ceph-mon[63158]: pgmap v151: 97 pgs: 92 active+clean, 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.3 MiB/s wr, 293 op/s; 23/1560 objects degraded (1.474%); 1.7 MiB/s, 33 objects/s recovering 2024-04-26T15:37:06.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:06 smithi164 ceph-mon[49952]: pgmap v151: 97 pgs: 92 active+clean, 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.3 MiB/s wr, 293 op/s; 23/1560 objects degraded (1.474%); 1.7 MiB/s, 33 objects/s recovering 2024-04-26T15:37:06.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 222550 10.66 MB/sec execute 390 sec latency 47.543 ms 2024-04-26T15:37:07.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:07 smithi112 ceph-mon[63158]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1560 objects degraded (1.474%), 1 pg degraded) 2024-04-26T15:37:07.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:07 smithi112 ceph-mon[63158]: Cluster is now healthy 2024-04-26T15:37:07.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:07 smithi164 ceph-mon[49952]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1560 objects degraded (1.474%), 1 pg degraded) 2024-04-26T15:37:07.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:07 smithi164 ceph-mon[49952]: Cluster is now healthy 2024-04-26T15:37:07.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 222814 10.65 MB/sec execute 391 sec latency 50.181 ms 2024-04-26T15:37:08.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:08 smithi112 ceph-mon[63158]: pgmap v152: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.7 MiB/s wr, 274 op/s; 202 KiB/s, 19 objects/s recovering 2024-04-26T15:37:08.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:08 smithi164 ceph-mon[49952]: pgmap v152: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.7 MiB/s wr, 274 op/s; 202 KiB/s, 19 objects/s recovering 2024-04-26T15:37:08.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 223302 10.66 MB/sec execute 392 sec latency 31.332 ms 2024-04-26T15:37:09.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 223789 10.66 MB/sec execute 393 sec latency 51.570 ms 2024-04-26T15:37:10.233 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:10 smithi112 ceph-mon[63158]: pgmap v153: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 2.6 MiB/s wr, 246 op/s; 169 KiB/s, 16 objects/s recovering 2024-04-26T15:37:10.234 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:10 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:10.234 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:10 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:37:10.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:10 smithi164 ceph-mon[49952]: pgmap v153: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 2.6 MiB/s wr, 246 op/s; 169 KiB/s, 16 objects/s recovering 2024-04-26T15:37:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:10 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:10 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:37:10.868 INFO:teuthology.orchestra.run.smithi112.stdout: 5 224344 10.66 MB/sec execute 394 sec latency 32.322 ms 2024-04-26T15:37:11.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 224971 10.68 MB/sec execute 395 sec latency 33.755 ms 2024-04-26T15:37:12.218 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:12 smithi112 ceph-mon[63158]: pgmap v154: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 337 op/s; 11 KiB/s, 8 objects/s recovering 2024-04-26T15:37:12.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:12 smithi164 ceph-mon[49952]: pgmap v154: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 337 op/s; 11 KiB/s, 8 objects/s recovering 2024-04-26T15:37:12.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 225455 10.69 MB/sec execute 396 sec latency 31.229 ms 2024-04-26T15:37:13.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 225834 10.69 MB/sec execute 397 sec latency 38.647 ms 2024-04-26T15:37:14.229 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:14 smithi112 ceph-mon[63158]: pgmap v155: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.8 MiB/s wr, 438 op/s; 9.7 KiB/s, 7 objects/s recovering 2024-04-26T15:37:14.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:14 smithi164 ceph-mon[49952]: pgmap v155: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.8 MiB/s wr, 438 op/s; 9.7 KiB/s, 7 objects/s recovering 2024-04-26T15:37:14.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 226210 10.69 MB/sec execute 398 sec latency 43.786 ms 2024-04-26T15:37:15.264 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:15 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:15.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:15 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:15.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 226448 10.68 MB/sec execute 399 sec latency 68.103 ms 2024-04-26T15:37:16.275 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:16 smithi112 ceph-mon[63158]: pgmap v156: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 5.6 MiB/s wr, 533 op/s; 0 B/s, 4 objects/s recovering 2024-04-26T15:37:16.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:16 smithi164 ceph-mon[49952]: pgmap v156: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 5.6 MiB/s wr, 533 op/s; 0 B/s, 4 objects/s recovering 2024-04-26T15:37:16.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 226661 10.67 MB/sec execute 400 sec latency 53.665 ms 2024-04-26T15:37:17.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 226872 10.66 MB/sec execute 401 sec latency 55.963 ms 2024-04-26T15:37:18.333 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:18 smithi112 ceph-mon[63158]: pgmap v157: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.7 MiB/s wr, 584 op/s; 0 B/s, 1 objects/s recovering 2024-04-26T15:37:18.333 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:18 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T15:37:18.333 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:18 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T15:37:18.333 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:18 smithi112 ceph-mon[63158]: Upgrade: osd.3 is safe to restart 2024-04-26T15:37:18.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:18 smithi164 ceph-mon[49952]: pgmap v157: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.7 MiB/s wr, 584 op/s; 0 B/s, 1 objects/s recovering 2024-04-26T15:37:18.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:18 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T15:37:18.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:18 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T15:37:18.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:18 smithi164 ceph-mon[49952]: Upgrade: osd.3 is safe to restart 2024-04-26T15:37:18.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 227115 10.65 MB/sec execute 402 sec latency 90.285 ms 2024-04-26T15:37:19.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 227409 10.64 MB/sec execute 403 sec latency 89.337 ms 2024-04-26T15:37:20.159 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:20 smithi112 ceph-mon[63158]: pgmap v158: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-04-26T15:37:20.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:20 smithi164 ceph-mon[49952]: pgmap v158: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-04-26T15:37:20.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 227722 10.62 MB/sec execute 404 sec latency 43.211 ms 2024-04-26T15:37:21.179 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:21 smithi112 ceph-mon[63158]: Upgrade: Updating osd.3 2024-04-26T15:37:21.179 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:21 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:21.180 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:21 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T15:37:21.180 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:21 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:37:21.180 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:21 smithi112 ceph-mon[63158]: Deploying daemon osd.3 on smithi112 2024-04-26T15:37:21.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:21 smithi164 ceph-mon[49952]: Upgrade: Updating osd.3 2024-04-26T15:37:21.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:21 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:21.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:21 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T15:37:21.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:21 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:37:21.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:21 smithi164 ceph-mon[49952]: Deploying daemon osd.3 on smithi112 2024-04-26T15:37:21.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 228007 10.61 MB/sec execute 405 sec latency 50.167 ms 2024-04-26T15:37:22.184 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:22 smithi112 ceph-mon[63158]: pgmap v159: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.4 MiB/s wr, 649 op/s 2024-04-26T15:37:22.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:22 smithi164 ceph-mon[49952]: pgmap v159: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.4 MiB/s wr, 649 op/s 2024-04-26T15:37:22.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 228360 10.62 MB/sec execute 406 sec latency 55.739 ms 2024-04-26T15:37:23.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 228681 10.60 MB/sec execute 407 sec latency 59.245 ms 2024-04-26T15:37:24.237 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:24 smithi112 ceph-mon[63158]: pgmap v160: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 556 op/s 2024-04-26T15:37:24.237 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:37:24.321 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:37:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:24 smithi164 ceph-mon[49952]: pgmap v160: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 556 op/s 2024-04-26T15:37:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:37:24.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 228947 10.60 MB/sec execute 408 sec latency 81.379 ms 2024-04-26T15:37:25.258 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:25 smithi112 ceph-mon[63158]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:25.258 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:25 smithi112 ceph-mon[63158]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:25.478 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:37:25.478 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (6m) 29s ago 15m 24.6M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (2m) 29s ago 15m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (2m) 2m ago 14m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (6m) 29s ago 15m 53.5M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (9m) 29s ago 9m 38.2M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (9m) 2m ago 9m 14.0M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (5m) 29s ago 16m 520M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (4m) 2m ago 14m 465M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (4m) 29s ago 16m 50.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (3m) 2m ago 14m 35.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (9m) 29s ago 9m 261M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (8m) 2m ago 8m 87.6M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (6m) 29s ago 15m 17.9M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (5m) 2m ago 14m 17.6M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:37:25.479 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (2m) 29s ago 13m 299M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:37:25.480 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (89s) 29s ago 13m 232M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:37:25.480 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (39s) 29s ago 12m 12.7M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:37:25.480 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (12m) 29s ago 12m 422M 1936M 17.2.0 e1d6a67b021e 75c57d9251c2 2024-04-26T15:37:25.480 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (12m) 2m ago 12m 551M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:37:25.480 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (11m) 2m ago 11m 373M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:37:25.480 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (11m) 2m ago 11m 557M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:37:25.480 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (10m) 2m ago 10m 448M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:37:25.480 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (4m) 29s ago 14m 73.1M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:37:25.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:25 smithi164 ceph-mon[49952]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:25.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:25 smithi164 ceph-mon[49952]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:25.769 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:25 smithi112 systemd[1]: Stopping Ceph osd.3 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:37:25.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 229142 10.59 MB/sec execute 409 sec latency 77.686 ms 2024-04-26T15:37:26.022 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:25 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3[42843]: 2024-04-26T15:37:25.928+0000 7f28d69b5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:37:26.022 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:25 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3[42843]: 2024-04-26T15:37:25.928+0000 7f28d69b5700 -1 osd.3 103 *** Got signal Terminated *** 2024-04-26T15:37:26.022 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:25 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3[42843]: 2024-04-26T15:37:25.928+0000 7f28d69b5700 -1 osd.3 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:37:26.159 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:37:26.160 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:37:26.160 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:37:26.160 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-26T15:37:26.160 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 7 2024-04-26T15:37:26.160 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:37:26.160 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:37:26.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:26 smithi112 ceph-mon[63158]: pgmap v161: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.1 MiB/s wr, 539 op/s 2024-04-26T15:37:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:26 smithi112 ceph-mon[63158]: from='client.44221 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:26 smithi112 ceph-mon[63158]: osd.3 marked itself down and dead 2024-04-26T15:37:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:26 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/2626971308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:26.525 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:37:26.525 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:37:26.525 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:37:26.526 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:37:26.526 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:37:26.526 INFO:teuthology.orchestra.run.smithi112.stdout: "crash", 2024-04-26T15:37:26.526 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:37:26.526 INFO:teuthology.orchestra.run.smithi112.stdout: "mon" 2024-04-26T15:37:26.526 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:37:26.526 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "9/23 daemons upgraded", 2024-04-26T15:37:26.526 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T15:37:26.526 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:37:26.526 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:37:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:26 smithi164 ceph-mon[49952]: pgmap v161: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.1 MiB/s wr, 539 op/s 2024-04-26T15:37:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:26 smithi164 ceph-mon[49952]: from='client.44221 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:26 smithi164 ceph-mon[49952]: osd.3 marked itself down and dead 2024-04-26T15:37:26.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:26 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/2626971308' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:26.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 229214 10.57 MB/sec execute 410 sec latency 692.712 ms 2024-04-26T15:37:26.944 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_WARN 1 osds down 2024-04-26T15:37:26.945 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-26T15:37:26.945 INFO:teuthology.orchestra.run.smithi112.stdout: osd.3 (root=default,host=smithi112) is down 2024-04-26T15:37:27.136 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:26 smithi112 podman[76750]: 2024-04-26 15:37:26.843208943 +0000 UTC m=+1.082061859 container died 75c57d9251c23d6e4d0b20acdcf844d4eeba1fff0332faa897e96a67deb33e63 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.openshift.expose-services=, ceph=True, CEPH_POINT_RELEASE=-17.2.0, version=8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , name=centos-stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, distribution-scope=public) 2024-04-26T15:37:27.386 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:27 smithi112 ceph-mon[63158]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:37:27.387 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:27 smithi112 ceph-mon[63158]: osdmap e104: 8 total, 7 up, 8 in 2024-04-26T15:37:27.387 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:27 smithi112 ceph-mon[63158]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:27.387 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:27 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/2889844156' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:37:27.387 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:27 smithi112 podman[76750]: 2024-04-26 15:37:27.344115475 +0000 UTC m=+1.582968402 container cleanup 75c57d9251c23d6e4d0b20acdcf844d4eeba1fff0332faa897e96a67deb33e63 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3, distribution-scope=public, release=754, maintainer=Guillaume Abrioux , version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, vcs-type=git, io.buildah.version=1.19.8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream) 2024-04-26T15:37:27.387 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:27 smithi112 bash[76750]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3 2024-04-26T15:37:27.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:27 smithi164 ceph-mon[49952]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:37:27.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:27 smithi164 ceph-mon[49952]: osdmap e104: 8 total, 7 up, 8 in 2024-04-26T15:37:27.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:27 smithi164 ceph-mon[49952]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:27.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:27 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/2889844156' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:37:27.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 229410 10.56 MB/sec execute 411 sec latency 1051.332 ms 2024-04-26T15:37:27.888 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:27 smithi112 podman[76811]: 2024-04-26 15:37:27.689940439 +0000 UTC m=+0.832932516 container remove 75c57d9251c23d6e4d0b20acdcf844d4eeba1fff0332faa897e96a67deb33e63 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, version=8, vcs-type=git, ceph=True, name=centos-stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_CLEAN=True, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64) 2024-04-26T15:37:28.145 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:28 smithi112 podman[76901]: 2024-04-26 15:37:28.010697078 +0000 UTC m=+0.026882972 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:37:28.396 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:28 smithi112 podman[76901]: 2024-04-26 15:37:28.186714784 +0000 UTC m=+0.202900670 container create 84f40c002e5770f485c84302cb3c81339de099510ce40e86cf136bd0a9b7c3b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-26T15:37:28.397 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:28 smithi112 ceph-mon[63158]: pgmap v163: 97 pgs: 16 stale+active+clean, 81 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.0 MiB/s wr, 421 op/s 2024-04-26T15:37:28.397 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:28 smithi112 ceph-mon[63158]: osdmap e105: 8 total, 7 up, 8 in 2024-04-26T15:37:28.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:28 smithi164 ceph-mon[49952]: pgmap v163: 97 pgs: 16 stale+active+clean, 81 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.0 MiB/s wr, 421 op/s 2024-04-26T15:37:28.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:28 smithi164 ceph-mon[49952]: osdmap e105: 8 total, 7 up, 8 in 2024-04-26T15:37:28.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 229845 10.57 MB/sec execute 412 sec latency 47.415 ms 2024-04-26T15:37:29.157 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:29 smithi112 podman[76901]: 2024-04-26 15:37:29.035191286 +0000 UTC m=+1.051377170 container init 84f40c002e5770f485c84302cb3c81339de099510ce40e86cf136bd0a9b7c3b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-26T15:37:29.158 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:29 smithi112 podman[76901]: 2024-04-26 15:37:29.042805527 +0000 UTC m=+1.058991416 container start 84f40c002e5770f485c84302cb3c81339de099510ce40e86cf136bd0a9b7c3b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:37:29.410 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:29 smithi112 podman[76901]: 2024-04-26 15:37:29.297155178 +0000 UTC m=+1.313341071 container attach 84f40c002e5770f485c84302cb3c81339de099510ce40e86cf136bd0a9b7c3b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS) 2024-04-26T15:37:29.411 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:29 smithi112 ceph-mon[63158]: Health check failed: Degraded data redundancy: 56/1614 objects degraded (3.470%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:29.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:29 smithi164 ceph-mon[49952]: Health check failed: Degraded data redundancy: 56/1614 objects degraded (3.470%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:29.667 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:29 smithi112 podman[76901]: 2024-04-26 15:37:29.51745251 +0000 UTC m=+1.533638397 container died 84f40c002e5770f485c84302cb3c81339de099510ce40e86cf136bd0a9b7c3b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-26T15:37:29.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 230242 10.57 MB/sec execute 413 sec latency 56.801 ms 2024-04-26T15:37:30.432 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:30 smithi112 ceph-mon[63158]: pgmap v165: 97 pgs: 2 active+undersized, 7 active+undersized+degraded, 13 stale+active+clean, 75 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.2 MiB/s wr, 308 op/s; 56/1614 objects degraded (3.470%) 2024-04-26T15:37:30.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:30 smithi164 ceph-mon[49952]: pgmap v165: 97 pgs: 2 active+undersized, 7 active+undersized+degraded, 13 stale+active+clean, 75 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.2 MiB/s wr, 308 op/s; 56/1614 objects degraded (3.470%) 2024-04-26T15:37:30.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 230553 10.57 MB/sec execute 414 sec latency 52.974 ms 2024-04-26T15:37:30.943 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:30 smithi112 podman[76927]: 2024-04-26 15:37:30.848601083 +0000 UTC m=+1.325114880 container remove 84f40c002e5770f485c84302cb3c81339de099510ce40e86cf136bd0a9b7c3b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:37:30.944 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:30 smithi112 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-26T15:37:30.944 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:30 smithi112 systemd[1]: Stopped Ceph osd.3 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:37:30.944 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:30 smithi112 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.3.service: Consumed 4min 2.700s CPU time. 2024-04-26T15:37:31.239 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:31 smithi112 systemd[1]: Starting Ceph osd.3 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:37:31.754 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:31 smithi112 podman[77022]: 2024-04-26 15:37:31.40963842 +0000 UTC m=+0.026147134 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:37:31.754 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:31 smithi112 podman[77022]: 2024-04-26 15:37:31.545023412 +0000 UTC m=+0.161532113 container create a044a8cf0c6d0e4cf97a23af9a841175ae314535ba3fe9c1532395d4e3ba0f1b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True) 2024-04-26T15:37:31.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 230878 10.56 MB/sec execute 415 sec latency 73.659 ms 2024-04-26T15:37:32.011 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:31 smithi112 podman[77022]: 2024-04-26 15:37:31.893831712 +0000 UTC m=+0.510340412 container init a044a8cf0c6d0e4cf97a23af9a841175ae314535ba3fe9c1532395d4e3ba0f1b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-26T15:37:32.012 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:31 smithi112 podman[77022]: 2024-04-26 15:37:31.901102592 +0000 UTC m=+0.517611285 container start a044a8cf0c6d0e4cf97a23af9a841175ae314535ba3fe9c1532395d4e3ba0f1b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS) 2024-04-26T15:37:32.012 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:31 smithi112 podman[77022]: 2024-04-26 15:37:31.993830675 +0000 UTC m=+0.610339382 container attach a044a8cf0c6d0e4cf97a23af9a841175ae314535ba3fe9c1532395d4e3ba0f1b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD) 2024-04-26T15:37:32.262 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:32 smithi112 ceph-mon[63158]: pgmap v166: 97 pgs: 10 active+undersized, 21 active+undersized+degraded, 4 stale+active+clean, 62 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 383 op/s; 181/1701 objects degraded (10.641%); 258 B/s, 0 objects/s recovering 2024-04-26T15:37:32.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:32 smithi164 ceph-mon[49952]: pgmap v166: 97 pgs: 10 active+undersized, 21 active+undersized+degraded, 4 stale+active+clean, 62 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 383 op/s; 181/1701 objects degraded (10.641%); 258 B/s, 0 objects/s recovering 2024-04-26T15:37:32.765 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate[77036]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:37:32.765 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate[77036]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:37:32.765 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 bash[77022]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:37:32.765 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 bash[77022]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:37:32.765 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 bash[77022]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T15:37:32.765 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 bash[77022]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T15:37:32.765 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 bash[77022]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T15:37:32.765 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 bash[77022]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T15:37:32.765 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 bash[77022]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T15:37:32.765 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 bash[77022]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T15:37:32.765 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 bash[77022]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T15:37:32.765 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate[77036]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T15:37:32.766 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate[77036]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T15:37:32.766 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate[77036]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T15:37:32.766 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate[77036]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T15:37:32.766 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate[77036]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T15:37:32.766 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate[77036]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T15:37:32.766 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate[77036]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T15:37:32.766 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:32 smithi112 podman[77022]: 2024-04-26 15:37:32.623197366 +0000 UTC m=+1.239706067 container died a044a8cf0c6d0e4cf97a23af9a841175ae314535ba3fe9c1532395d4e3ba0f1b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True) 2024-04-26T15:37:32.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 231227 10.54 MB/sec execute 416 sec latency 61.607 ms 2024-04-26T15:37:33.785 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:33 smithi112 podman[77174]: 2024-04-26 15:37:33.560037889 +0000 UTC m=+0.924863332 container remove a044a8cf0c6d0e4cf97a23af9a841175ae314535ba3fe9c1532395d4e3ba0f1b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-26T15:37:33.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 231490 10.53 MB/sec execute 417 sec latency 66.140 ms 2024-04-26T15:37:34.039 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:33 smithi112 podman[77230]: 2024-04-26 15:37:33.720914068 +0000 UTC m=+0.027249445 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:37:34.040 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:33 smithi112 podman[77230]: 2024-04-26 15:37:33.910570835 +0000 UTC m=+0.216906200 container create 27fefb9c15b304ec42d0dc5082025426dd4bd93ea0dd19a7521d1faf1ac35e1a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=) 2024-04-26T15:37:34.299 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:34 smithi112 ceph-mon[63158]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.3 MiB/s wr, 415 op/s; 220/1773 objects degraded (12.408%); 258 B/s, 0 objects/s recovering 2024-04-26T15:37:34.558 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:34 smithi112 podman[77230]: 2024-04-26 15:37:34.343490527 +0000 UTC m=+0.649825899 container init 27fefb9c15b304ec42d0dc5082025426dd4bd93ea0dd19a7521d1faf1ac35e1a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=) 2024-04-26T15:37:34.558 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:34 smithi112 podman[77230]: 2024-04-26 15:37:34.351557248 +0000 UTC m=+0.657892611 container start 27fefb9c15b304ec42d0dc5082025426dd4bd93ea0dd19a7521d1faf1ac35e1a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux ) 2024-04-26T15:37:34.558 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:34 smithi112 ceph-osd[77248]: -- 172.21.15.112:0/2359731498 <== mon.0 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 357+0+0 (secure 0 0 0) 0x56073e9c7d40 con 0x56073e999000 2024-04-26T15:37:34.558 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:34 smithi112 ceph-osd[77248]: -- 172.21.15.112:0/2359731498 <== mon.0 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56073e9cf680 con 0x56073e999000 2024-04-26T15:37:34.558 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:34 smithi112 bash[77230]: 27fefb9c15b304ec42d0dc5082025426dd4bd93ea0dd19a7521d1faf1ac35e1a 2024-04-26T15:37:34.558 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:34 smithi112 systemd[1]: Started Ceph osd.3 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:37:34.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:34 smithi164 ceph-mon[49952]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.3 MiB/s wr, 415 op/s; 220/1773 objects degraded (12.408%); 258 B/s, 0 objects/s recovering 2024-04-26T15:37:34.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 231841 10.53 MB/sec execute 418 sec latency 54.556 ms 2024-04-26T15:37:35.063 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:34 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3[77244]: 2024-04-26T15:37:34.922+0000 7f0d0af0d7c0 -1 Falling back to public interface 2024-04-26T15:37:35.326 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:35 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 220/1773 objects degraded (12.408%), 27 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:35.327 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:35 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:35.327 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:35 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:35.327 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:35 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:37:35.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:35 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 220/1773 objects degraded (12.408%), 27 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:35.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:35 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:35.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:35 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:35.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:35 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:37:35.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 232218 10.53 MB/sec execute 419 sec latency 58.779 ms 2024-04-26T15:37:36.606 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:36 smithi112 ceph-mon[63158]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s wr, 360 op/s; 212/1674 objects degraded (12.664%); 233 B/s, 0 objects/s recovering 2024-04-26T15:37:36.869 INFO:teuthology.orchestra.run.smithi112.stdout: 5 232561 10.53 MB/sec execute 420 sec latency 47.705 ms 2024-04-26T15:37:37.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:36 smithi164 ceph-mon[49952]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s wr, 360 op/s; 212/1674 objects degraded (12.664%); 233 B/s, 0 objects/s recovering 2024-04-26T15:37:37.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 232892 10.53 MB/sec execute 421 sec latency 51.477 ms 2024-04-26T15:37:38.655 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:38 smithi112 ceph-mon[63158]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.3 MiB/s wr, 433 op/s; 213/1662 objects degraded (12.816%); 206 B/s, 0 objects/s recovering 2024-04-26T15:37:38.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 233191 10.53 MB/sec execute 422 sec latency 44.670 ms 2024-04-26T15:37:39.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:38 smithi164 ceph-mon[49952]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.3 MiB/s wr, 433 op/s; 213/1662 objects degraded (12.816%); 206 B/s, 0 objects/s recovering 2024-04-26T15:37:39.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 233489 10.52 MB/sec execute 423 sec latency 43.916 ms 2024-04-26T15:37:40.190 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:40 smithi112 ceph-mon[63158]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.9 MiB/s wr, 384 op/s; 212/1638 objects degraded (12.943%); 174 B/s, 0 objects/s recovering 2024-04-26T15:37:40.190 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:40.191 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:37:40.191 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:40 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 213/1662 objects degraded (12.816%), 27 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:40.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:40 smithi164 ceph-mon[49952]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.9 MiB/s wr, 384 op/s; 212/1638 objects degraded (12.943%); 174 B/s, 0 objects/s recovering 2024-04-26T15:37:40.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:40.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:37:40.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:40 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 213/1662 objects degraded (12.816%), 27 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:40.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 233751 10.52 MB/sec execute 424 sec latency 47.380 ms 2024-04-26T15:37:41.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 233999 10.51 MB/sec execute 425 sec latency 50.788 ms 2024-04-26T15:37:42.217 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:42 smithi112 ceph-mon[63158]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.2 MiB/s wr, 537 op/s; 216/1671 objects degraded (12.926%); 172 B/s, 0 objects/s recovering 2024-04-26T15:37:42.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:42 smithi164 ceph-mon[49952]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.2 MiB/s wr, 537 op/s; 216/1671 objects degraded (12.926%); 172 B/s, 0 objects/s recovering 2024-04-26T15:37:42.733 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:42 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3[77244]: 2024-04-26T15:37:42.661+0000 7f0d0af0d7c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-26T15:37:42.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 234328 10.51 MB/sec execute 426 sec latency 53.789 ms 2024-04-26T15:37:43.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 234682 10.49 MB/sec execute 427 sec latency 40.813 ms 2024-04-26T15:37:44.310 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3[77244]: 2024-04-26T15:37:44.033+0000 7f0d0af0d7c0 -1 osd.3 103 log_to_monitors true 2024-04-26T15:37:44.311 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:44 smithi112 ceph-mon[63158]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.9 MiB/s wr, 490 op/s; 216/1659 objects degraded (13.020%) 2024-04-26T15:37:44.311 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:44 smithi112 ceph-mon[63158]: from='osd.3 [v2:172.21.15.112:6826/1755862344,v1:172.21.15.112:6827/1755862344]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T15:37:44.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:44 smithi164 ceph-mon[49952]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.9 MiB/s wr, 490 op/s; 216/1659 objects degraded (13.020%) 2024-04-26T15:37:44.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:44 smithi164 ceph-mon[49952]: from='osd.3 [v2:172.21.15.112:6826/1755862344,v1:172.21.15.112:6827/1755862344]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T15:37:44.814 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:37:44 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3[77244]: 2024-04-26T15:37:44.631+0000 7f0cf835d700 -1 osd.3 103 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:37:44.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 234901 10.48 MB/sec execute 428 sec latency 133.499 ms 2024-04-26T15:37:45.318 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:45 smithi112 ceph-mon[63158]: from='osd.3 [v2:172.21.15.112:6826/1755862344,v1:172.21.15.112:6827/1755862344]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T15:37:45.318 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:45 smithi112 ceph-mon[63158]: osdmap e106: 8 total, 7 up, 8 in 2024-04-26T15:37:45.318 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:45 smithi112 ceph-mon[63158]: from='osd.3 [v2:172.21.15.112:6826/1755862344,v1:172.21.15.112:6827/1755862344]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:37:45.318 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:45.318 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:45 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 216/1659 objects degraded (13.020%), 27 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:45.318 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:45.318 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:45 smithi112 ceph-mon[63158]: from='osd.3 [v2:172.21.15.112:6826/1755862344,v1:172.21.15.112:6827/1755862344]' entity='osd.3' 2024-04-26T15:37:45.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:45 smithi164 ceph-mon[49952]: from='osd.3 [v2:172.21.15.112:6826/1755862344,v1:172.21.15.112:6827/1755862344]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T15:37:45.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:45 smithi164 ceph-mon[49952]: osdmap e106: 8 total, 7 up, 8 in 2024-04-26T15:37:45.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:45 smithi164 ceph-mon[49952]: from='osd.3 [v2:172.21.15.112:6826/1755862344,v1:172.21.15.112:6827/1755862344]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-26T15:37:45.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:45.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:45 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 216/1659 objects degraded (13.020%), 27 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:45.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:45.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:45 smithi164 ceph-mon[49952]: from='osd.3 [v2:172.21.15.112:6826/1755862344,v1:172.21.15.112:6827/1755862344]' entity='osd.3' 2024-04-26T15:37:45.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 235178 10.47 MB/sec execute 429 sec latency 240.838 ms 2024-04-26T15:37:46.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:46 smithi112 ceph-mon[63158]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 503 op/s; 207/1623 objects degraded (12.754%) 2024-04-26T15:37:46.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:46 smithi112 ceph-mon[63158]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:37:46.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:46 smithi112 ceph-mon[63158]: osd.3 [v2:172.21.15.112:6826/1755862344,v1:172.21.15.112:6827/1755862344] boot 2024-04-26T15:37:46.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:46 smithi112 ceph-mon[63158]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T15:37:46.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:46 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:37:46.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:46 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:46.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:46 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:46.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:46 smithi164 ceph-mon[49952]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 503 op/s; 207/1623 objects degraded (12.754%) 2024-04-26T15:37:46.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:46 smithi164 ceph-mon[49952]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:37:46.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:46 smithi164 ceph-mon[49952]: osd.3 [v2:172.21.15.112:6826/1755862344,v1:172.21.15.112:6827/1755862344] boot 2024-04-26T15:37:46.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:46 smithi164 ceph-mon[49952]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T15:37:46.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:46 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:37:46.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:46 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:46.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:46 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:46.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 235178 10.45 MB/sec execute 430 sec latency 1240.866 ms 2024-04-26T15:37:47.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:47 smithi112 ceph-mon[63158]: osdmap e108: 8 total, 8 up, 8 in 2024-04-26T15:37:47.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 235178 10.42 MB/sec execute 431 sec latency 2240.896 ms 2024-04-26T15:37:48.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:47 smithi164 ceph-mon[49952]: osdmap e108: 8 total, 8 up, 8 in 2024-04-26T15:37:48.654 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:48 smithi112 ceph-mon[63158]: pgmap v177: 97 pgs: 6 peering, 12 active+undersized, 23 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.1 MiB/s wr, 388 op/s; 189/1632 objects degraded (11.581%) 2024-04-26T15:37:48.655 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:48 smithi112 ceph-mon[63158]: osdmap e109: 8 total, 8 up, 8 in 2024-04-26T15:37:48.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 235401 10.41 MB/sec execute 432 sec latency 2435.144 ms 2024-04-26T15:37:49.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:48 smithi164 ceph-mon[49952]: pgmap v177: 97 pgs: 6 peering, 12 active+undersized, 23 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.1 MiB/s wr, 388 op/s; 189/1632 objects degraded (11.581%) 2024-04-26T15:37:49.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:48 smithi164 ceph-mon[49952]: osdmap e109: 8 total, 8 up, 8 in 2024-04-26T15:37:49.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:49 smithi112 ceph-mon[63158]: osdmap e110: 8 total, 8 up, 8 in 2024-04-26T15:37:49.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:49 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 189/1632 objects degraded (11.581%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:49.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:49 smithi112 ceph-mon[63158]: osdmap e111: 8 total, 8 up, 8 in 2024-04-26T15:37:49.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 235536 10.40 MB/sec execute 433 sec latency 749.825 ms 2024-04-26T15:37:50.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:49 smithi164 ceph-mon[49952]: osdmap e110: 8 total, 8 up, 8 in 2024-04-26T15:37:50.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:49 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 189/1632 objects degraded (11.581%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:50.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:49 smithi164 ceph-mon[49952]: osdmap e111: 8 total, 8 up, 8 in 2024-04-26T15:37:50.761 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: pgmap v180: 97 pgs: 2 activating+undersized+degraded+remapped, 6 peering, 10 active+undersized, 18 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 1.7 MiB/s wr, 188 op/s; 176/1599 objects degraded (11.007%); 74 KiB/s, 7 objects/s recovering 2024-04-26T15:37:50.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:50.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:50.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:37:50.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:37:50.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:50.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:50.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:37:50.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:50.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:50.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:50.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T15:37:50.763 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:50 smithi112 ceph-mon[63158]: osdmap e112: 8 total, 8 up, 8 in 2024-04-26T15:37:50.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 235864 10.40 MB/sec execute 434 sec latency 1010.566 ms 2024-04-26T15:37:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: pgmap v180: 97 pgs: 2 activating+undersized+degraded+remapped, 6 peering, 10 active+undersized, 18 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 1.7 MiB/s wr, 188 op/s; 176/1599 objects degraded (11.007%); 74 KiB/s, 7 objects/s recovering 2024-04-26T15:37:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:37:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:37:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:37:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:51.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:51.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T15:37:51.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:50 smithi164 ceph-mon[49952]: osdmap e112: 8 total, 8 up, 8 in 2024-04-26T15:37:51.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:51 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T15:37:51.837 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:51 smithi112 ceph-mon[63158]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-26T15:37:51.838 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:51 smithi112 ceph-mon[63158]: osdmap e113: 8 total, 8 up, 8 in 2024-04-26T15:37:51.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 236121 10.39 MB/sec execute 435 sec latency 509.305 ms 2024-04-26T15:37:52.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:51 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T15:37:52.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:51 smithi164 ceph-mon[49952]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-26T15:37:52.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:51 smithi164 ceph-mon[49952]: osdmap e113: 8 total, 8 up, 8 in 2024-04-26T15:37:52.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 236464 10.40 MB/sec execute 436 sec latency 1013.813 ms 2024-04-26T15:37:52.873 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:52 smithi112 ceph-mon[63158]: pgmap v183: 97 pgs: 1 activating+undersized+degraded+remapped, 9 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 85 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.6 MiB/s wr, 218 op/s; 39/1506 objects degraded (2.590%); 700 KiB/s, 38 objects/s recovering 2024-04-26T15:37:52.873 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:52 smithi112 ceph-mon[63158]: osdmap e114: 8 total, 8 up, 8 in 2024-04-26T15:37:53.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:52 smithi164 ceph-mon[49952]: pgmap v183: 97 pgs: 1 activating+undersized+degraded+remapped, 9 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 85 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.6 MiB/s wr, 218 op/s; 39/1506 objects degraded (2.590%); 700 KiB/s, 38 objects/s recovering 2024-04-26T15:37:53.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:52 smithi164 ceph-mon[49952]: osdmap e114: 8 total, 8 up, 8 in 2024-04-26T15:37:53.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 236881 10.40 MB/sec execute 437 sec latency 56.148 ms 2024-04-26T15:37:54.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 237187 10.39 MB/sec execute 438 sec latency 43.415 ms 2024-04-26T15:37:54.920 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:54 smithi112 ceph-mon[63158]: pgmap v186: 97 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 92 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 2.1 MiB/s wr, 234 op/s; 14/1518 objects degraded (0.922%); 723 KiB/s, 42 objects/s recovering 2024-04-26T15:37:54.920 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:54.920 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:37:54.920 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:54 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 14/1518 objects degraded (0.922%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:54 smithi164 ceph-mon[49952]: pgmap v186: 97 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 92 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 2.1 MiB/s wr, 234 op/s; 14/1518 objects degraded (0.922%); 723 KiB/s, 42 objects/s recovering 2024-04-26T15:37:55.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:37:55.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:37:55.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:54 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 14/1518 objects degraded (0.922%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T15:37:55.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 237497 10.39 MB/sec execute 439 sec latency 53.818 ms 2024-04-26T15:37:55.932 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:55 smithi112 ceph-mon[63158]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/1518 objects degraded (0.922%), 2 pgs degraded) 2024-04-26T15:37:55.933 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:55 smithi112 ceph-mon[63158]: Cluster is now healthy 2024-04-26T15:37:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:55 smithi164 ceph-mon[49952]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/1518 objects degraded (0.922%), 2 pgs degraded) 2024-04-26T15:37:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:55 smithi164 ceph-mon[49952]: Cluster is now healthy 2024-04-26T15:37:56.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 237922 10.38 MB/sec execute 440 sec latency 52.184 ms 2024-04-26T15:37:56.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:56 smithi112 ceph-mon[63158]: pgmap v187: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.1 KiB/s rd, 3.7 MiB/s wr, 244 op/s; 508 KiB/s, 32 objects/s recovering 2024-04-26T15:37:57.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:56 smithi164 ceph-mon[49952]: pgmap v187: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.1 KiB/s rd, 3.7 MiB/s wr, 244 op/s; 508 KiB/s, 32 objects/s recovering 2024-04-26T15:37:57.479 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:37:57.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 238230 10.38 MB/sec execute 441 sec latency 47.702 ms 2024-04-26T15:37:58.496 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:37:58.497 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (6m) 14s ago 15m 25.1M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:37:58.497 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (3m) 14s ago 15m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:37:58.497 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (3m) 3m ago 15m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:37:58.497 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (6m) 14s ago 15m 54.1M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:37:58.497 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (9m) 14s ago 9m 38.3M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:37:58.497 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (9m) 3m ago 9m 14.0M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:37:58.497 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (6m) 14s ago 17m 522M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:37:58.497 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (5m) 3m ago 15m 465M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:37:58.497 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (4m) 14s ago 17m 52.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:37:58.497 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (4m) 3m ago 15m 35.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:37:58.497 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (9m) 14s ago 9m 266M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:37:58.497 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (9m) 3m ago 9m 87.6M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:37:58.498 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (6m) 14s ago 15m 17.5M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:37:58.498 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (6m) 3m ago 14m 17.6M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:37:58.498 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (2m) 14s ago 14m 353M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:37:58.498 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (2m) 14s ago 13m 291M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:37:58.498 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (72s) 14s ago 13m 213M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:37:58.498 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (24s) 14s ago 12m 14.9M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd 27fefb9c15b3 2024-04-26T15:37:58.498 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (12m) 3m ago 12m 551M 2704M 17.2.0 e1d6a67b021e ea5fd5f70877 2024-04-26T15:37:58.498 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (12m) 3m ago 12m 373M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:37:58.498 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (11m) 3m ago 11m 557M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:37:58.498 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (11m) 3m ago 11m 448M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:37:58.498 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (5m) 14s ago 14m 73.2M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:37:58.731 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:58 smithi112 ceph-mon[63158]: pgmap v188: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.2 MiB/s wr, 364 op/s; 77 KiB/s, 14 objects/s recovering 2024-04-26T15:37:58.731 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:58 smithi112 ceph-mon[63158]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:58.731 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:58 smithi112 ceph-mon[63158]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:58.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 238557 10.37 MB/sec execute 442 sec latency 63.073 ms 2024-04-26T15:37:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:58 smithi164 ceph-mon[49952]: pgmap v188: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.2 MiB/s wr, 364 op/s; 77 KiB/s, 14 objects/s recovering 2024-04-26T15:37:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:58 smithi164 ceph-mon[49952]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:58 smithi164 ceph-mon[49952]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:59.226 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:37:59.226 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:37:59.226 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:37:59.226 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 4 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T15:37:59.227 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:37:59.228 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:37:59.754 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:59 smithi112 ceph-mon[63158]: from='client.44243 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:37:59.754 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:37:59 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/977086635' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:37:59.759 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:37:59.759 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:37:59.759 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:37:59.759 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:37:59.759 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:37:59.760 INFO:teuthology.orchestra.run.smithi112.stdout: "crash", 2024-04-26T15:37:59.760 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:37:59.760 INFO:teuthology.orchestra.run.smithi112.stdout: "mon" 2024-04-26T15:37:59.760 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:37:59.760 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "10/23 daemons upgraded", 2024-04-26T15:37:59.760 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T15:37:59.760 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:37:59.760 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:37:59.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 238849 10.37 MB/sec execute 443 sec latency 58.773 ms 2024-04-26T15:38:00.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:59 smithi164 ceph-mon[49952]: from='client.44243 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:38:00.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:37:59 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/977086635' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:38:00.395 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:38:00.761 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:00 smithi112 ceph-mon[63158]: pgmap v189: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.8 MiB/s wr, 321 op/s; 65 KiB/s, 12 objects/s recovering 2024-04-26T15:38:00.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:00 smithi112 ceph-mon[63158]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:38:00.762 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:00 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/749521836' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:38:00.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 239245 10.37 MB/sec execute 444 sec latency 56.699 ms 2024-04-26T15:38:01.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:00 smithi164 ceph-mon[49952]: pgmap v189: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.8 MiB/s wr, 321 op/s; 65 KiB/s, 12 objects/s recovering 2024-04-26T15:38:01.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:00 smithi164 ceph-mon[49952]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:38:01.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:00 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/749521836' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:38:01.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 239546 10.36 MB/sec execute 445 sec latency 40.289 ms 2024-04-26T15:38:02.808 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:02 smithi112 ceph-mon[63158]: pgmap v190: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 438 op/s; 14 KiB/s, 8 objects/s recovering 2024-04-26T15:38:02.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 239724 10.35 MB/sec execute 446 sec latency 64.407 ms 2024-04-26T15:38:03.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:02 smithi164 ceph-mon[49952]: pgmap v190: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 438 op/s; 14 KiB/s, 8 objects/s recovering 2024-04-26T15:38:03.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 240040 10.36 MB/sec execute 447 sec latency 63.075 ms 2024-04-26T15:38:04.861 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:04 smithi112 ceph-mon[63158]: pgmap v191: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.1 MiB/s wr, 503 op/s; 13 KiB/s, 7 objects/s recovering 2024-04-26T15:38:04.862 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:04 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:04.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 240232 10.34 MB/sec execute 448 sec latency 85.317 ms 2024-04-26T15:38:05.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:04 smithi164 ceph-mon[49952]: pgmap v191: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.1 MiB/s wr, 503 op/s; 13 KiB/s, 7 objects/s recovering 2024-04-26T15:38:05.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:04 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:05.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 240499 10.33 MB/sec execute 449 sec latency 70.273 ms 2024-04-26T15:38:05.939 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T15:38:06.017 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T15:38:06.808 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:06 smithi112 ceph-mon[63158]: pgmap v192: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.4 MiB/s wr, 534 op/s; 11 KiB/s, 4 objects/s recovering 2024-04-26T15:38:06.808 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:06 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T15:38:06.808 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:06 smithi112 ceph-mon[63158]: Upgrade: osd.4 is safe to restart 2024-04-26T15:38:06.870 INFO:teuthology.orchestra.run.smithi112.stdout: 5 240744 10.33 MB/sec execute 450 sec latency 58.799 ms 2024-04-26T15:38:07.036 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:06 smithi164 ceph-mon[49952]: pgmap v192: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.4 MiB/s wr, 534 op/s; 11 KiB/s, 4 objects/s recovering 2024-04-26T15:38:07.037 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:06 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T15:38:07.037 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:06 smithi164 ceph-mon[49952]: Upgrade: osd.4 is safe to restart 2024-04-26T15:38:07.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 240989 10.32 MB/sec execute 451 sec latency 56.625 ms 2024-04-26T15:38:08.247 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:08 smithi112 ceph-mon[63158]: pgmap v193: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.7 MiB/s wr, 527 op/s 2024-04-26T15:38:08.248 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:08 smithi112 ceph-mon[63158]: Upgrade: Updating osd.4 2024-04-26T15:38:08.248 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:08.248 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T15:38:08.248 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:38:08.248 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:08 smithi112 ceph-mon[63158]: Deploying daemon osd.4 on smithi164 2024-04-26T15:38:08.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:08 smithi164 ceph-mon[49952]: pgmap v193: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.7 MiB/s wr, 527 op/s 2024-04-26T15:38:08.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:08 smithi164 ceph-mon[49952]: Upgrade: Updating osd.4 2024-04-26T15:38:08.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:08.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T15:38:08.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:38:08.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:08 smithi164 ceph-mon[49952]: Deploying daemon osd.4 on smithi164 2024-04-26T15:38:08.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 241384 10.31 MB/sec execute 452 sec latency 57.422 ms 2024-04-26T15:38:09.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 241704 10.31 MB/sec execute 453 sec latency 45.938 ms 2024-04-26T15:38:10.296 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:10 smithi112 ceph-mon[63158]: pgmap v194: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 387 op/s 2024-04-26T15:38:10.297 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:10 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:10.297 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:10 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:38:10.314 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:10 smithi164 ceph-mon[49952]: pgmap v194: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 387 op/s 2024-04-26T15:38:10.315 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:10 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:10.315 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:10 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:38:10.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 242092 10.31 MB/sec execute 454 sec latency 43.279 ms 2024-04-26T15:38:11.098 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:10 smithi164 systemd[1]: Stopping Ceph osd.4 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:38:11.597 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:11 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[28308]: 2024-04-26T15:38:11.286+0000 7f77434b3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:38:11.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:11 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[28308]: 2024-04-26T15:38:11.286+0000 7f77434b3700 -1 osd.4 114 *** Got signal Terminated *** 2024-04-26T15:38:11.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:11 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[28308]: 2024-04-26T15:38:11.286+0000 7f77434b3700 -1 osd.4 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:38:11.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 242474 10.31 MB/sec execute 455 sec latency 43.698 ms 2024-04-26T15:38:12.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:12 smithi112 ceph-mon[63158]: pgmap v195: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.7 MiB/s wr, 455 op/s 2024-04-26T15:38:12.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:12 smithi112 ceph-mon[63158]: osd.4 marked itself down and dead 2024-04-26T15:38:12.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:12 smithi164 ceph-mon[49952]: pgmap v195: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.7 MiB/s wr, 455 op/s 2024-04-26T15:38:12.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:12 smithi164 ceph-mon[49952]: osd.4 marked itself down and dead 2024-04-26T15:38:12.836 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:12 smithi164 podman[54439]: 2024-04-26 15:38:12.559073104 +0000 UTC m=+1.437839738 container died ea5fd5f70877cb2f5378c42d0d197c0b0584a3ec5ba9745d15fcfd0daa585026 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, name=centos-stream, release=754, io.openshift.expose-services=, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, ceph=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-26T15:38:12.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 242641 10.29 MB/sec execute 456 sec latency 773.061 ms 2024-04-26T15:38:13.092 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:12 smithi164 podman[54439]: 2024-04-26 15:38:12.935846485 +0000 UTC m=+1.814613128 container cleanup ea5fd5f70877cb2f5378c42d0d197c0b0584a3ec5ba9745d15fcfd0daa585026 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, distribution-scope=public, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, release=754, version=8) 2024-04-26T15:38:13.092 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:12 smithi164 bash[54439]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4 2024-04-26T15:38:13.347 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:13 smithi112 ceph-mon[63158]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:38:13.347 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:13 smithi112 ceph-mon[63158]: osdmap e115: 8 total, 7 up, 8 in 2024-04-26T15:38:13.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:13 smithi164 ceph-mon[49952]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:38:13.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:13 smithi164 ceph-mon[49952]: osdmap e115: 8 total, 7 up, 8 in 2024-04-26T15:38:13.348 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:13 smithi164 podman[54464]: 2024-04-26 15:38:13.212560379 +0000 UTC m=+0.651776652 container remove ea5fd5f70877cb2f5378c42d0d197c0b0584a3ec5ba9745d15fcfd0daa585026 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, name=centos-stream, version=8, RELEASE=HEAD, ceph=True, release=754, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream) 2024-04-26T15:38:13.865 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:13 smithi164 podman[54554]: 2024-04-26 15:38:13.506169789 +0000 UTC m=+0.020114606 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:38:13.865 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:13 smithi164 podman[54554]: 2024-04-26 15:38:13.638665296 +0000 UTC m=+0.152610096 container create dedec6155a9723d457aaad986c181e58de6870011479616ce61a3c6fb6d8cbc3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-deactivate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-26T15:38:13.865 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:13 smithi164 podman[54554]: 2024-04-26 15:38:13.866644904 +0000 UTC m=+0.380589705 container init dedec6155a9723d457aaad986c181e58de6870011479616ce61a3c6fb6d8cbc3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:38:13.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 242942 10.29 MB/sec execute 457 sec latency 1079.732 ms 2024-04-26T15:38:14.119 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:14 smithi112 ceph-mon[63158]: pgmap v197: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 395 op/s 2024-04-26T15:38:14.119 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:14 smithi112 ceph-mon[63158]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T15:38:14.119 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:14 smithi112 ceph-mon[63158]: osdmap e116: 8 total, 7 up, 8 in 2024-04-26T15:38:14.194 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:14 smithi164 ceph-mon[49952]: pgmap v197: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 395 op/s 2024-04-26T15:38:14.194 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:14 smithi164 ceph-mon[49952]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T15:38:14.194 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:14 smithi164 ceph-mon[49952]: osdmap e116: 8 total, 7 up, 8 in 2024-04-26T15:38:14.194 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:13 smithi164 podman[54554]: 2024-04-26 15:38:13.870431242 +0000 UTC m=+0.384376043 container start dedec6155a9723d457aaad986c181e58de6870011479616ce61a3c6fb6d8cbc3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T15:38:14.195 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:13 smithi164 podman[54554]: 2024-04-26 15:38:13.975186847 +0000 UTC m=+0.489131661 container attach dedec6155a9723d457aaad986c181e58de6870011479616ce61a3c6fb6d8cbc3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-26T15:38:14.553 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:14 smithi164 podman[54554]: 2024-04-26 15:38:14.1954917 +0000 UTC m=+0.709436501 container died dedec6155a9723d457aaad986c181e58de6870011479616ce61a3c6fb6d8cbc3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T15:38:14.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 243188 10.28 MB/sec execute 458 sec latency 53.133 ms 2024-04-26T15:38:15.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:15 smithi164 ceph-mon[49952]: Health check failed: Degraded data redundancy: 129/1638 objects degraded (7.875%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T15:38:15.348 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:14 smithi164 podman[54580]: 2024-04-26 15:38:14.947093339 +0000 UTC m=+0.745718991 container remove dedec6155a9723d457aaad986c181e58de6870011479616ce61a3c6fb6d8cbc3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:38:15.348 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:14 smithi164 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-26T15:38:15.348 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:14 smithi164 systemd[1]: Stopped Ceph osd.4 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:38:15.348 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:14 smithi164 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.4.service: Consumed 4min 28.529s CPU time. 2024-04-26T15:38:15.348 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:15 smithi164 systemd[1]: Starting Ceph osd.4 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:38:15.395 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:15 smithi112 ceph-mon[63158]: Health check failed: Degraded data redundancy: 129/1638 objects degraded (7.875%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T15:38:15.700 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:15 smithi164 podman[54675]: 2024-04-26 15:38:15.314323814 +0000 UTC m=+0.019076785 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:38:15.701 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:15 smithi164 podman[54675]: 2024-04-26 15:38:15.448735789 +0000 UTC m=+0.153488731 container create 8606a5f07983199a383fe6414e65addfaffb662432bf16bc4eb77ed5873945f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5) 2024-04-26T15:38:15.701 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:15 smithi164 podman[54675]: 2024-04-26 15:38:15.70148804 +0000 UTC m=+0.406240986 container init 8606a5f07983199a383fe6414e65addfaffb662432bf16bc4eb77ed5873945f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-26T15:38:15.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 243577 10.30 MB/sec execute 459 sec latency 44.834 ms 2024-04-26T15:38:16.097 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:15 smithi164 podman[54675]: 2024-04-26 15:38:15.70574012 +0000 UTC m=+0.410493068 container start 8606a5f07983199a383fe6414e65addfaffb662432bf16bc4eb77ed5873945f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True) 2024-04-26T15:38:16.098 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:15 smithi164 podman[54675]: 2024-04-26 15:38:15.810042802 +0000 UTC m=+0.514795774 container attach 8606a5f07983199a383fe6414e65addfaffb662432bf16bc4eb77ed5873945f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux ) 2024-04-26T15:38:16.408 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:16 smithi112 ceph-mon[63158]: pgmap v199: 97 pgs: 8 active+undersized, 15 active+undersized+degraded, 8 peering, 4 stale+active+clean, 62 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.6 MiB/s wr, 483 op/s; 129/1638 objects degraded (7.875%) 2024-04-26T15:38:16.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:16 smithi164 ceph-mon[49952]: pgmap v199: 97 pgs: 8 active+undersized, 15 active+undersized+degraded, 8 peering, 4 stale+active+clean, 62 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.6 MiB/s wr, 483 op/s; 129/1638 objects degraded (7.875%) 2024-04-26T15:38:16.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate[54690]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:38:16.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate[54690]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:38:16.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 bash[54675]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:38:16.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 bash[54675]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:38:16.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 bash[54675]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-26T15:38:16.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 bash[54675]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T15:38:16.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 bash[54675]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T15:38:16.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 bash[54675]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T15:38:16.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 bash[54675]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-26T15:38:16.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 bash[54675]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-26T15:38:16.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 bash[54675]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-26T15:38:16.599 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate[54690]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-26T15:38:16.599 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate[54690]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T15:38:16.599 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate[54690]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T15:38:16.599 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate[54690]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T15:38:16.599 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate[54690]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-26T15:38:16.599 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate[54690]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-26T15:38:16.599 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate[54690]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-26T15:38:16.599 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:16 smithi164 podman[54675]: 2024-04-26 15:38:16.262284985 +0000 UTC m=+0.967037941 container died 8606a5f07983199a383fe6414e65addfaffb662432bf16bc4eb77ed5873945f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424) 2024-04-26T15:38:16.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 243932 10.29 MB/sec execute 460 sec latency 49.956 ms 2024-04-26T15:38:17.357 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:17 smithi164 podman[54828]: 2024-04-26 15:38:17.107631366 +0000 UTC m=+0.839140949 container remove 8606a5f07983199a383fe6414e65addfaffb662432bf16bc4eb77ed5873945f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-26T15:38:17.357 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:17 smithi164 podman[54892]: 2024-04-26 15:38:17.229106208 +0000 UTC m=+0.019153632 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:38:17.627 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:17 smithi164 podman[54892]: 2024-04-26 15:38:17.358712506 +0000 UTC m=+0.148759915 container create b68352a89f2e5137c025a2e2a6556c51a993656435747fb9a79467ccf76f2c57 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424) 2024-04-26T15:38:17.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 244223 10.29 MB/sec execute 461 sec latency 60.575 ms 2024-04-26T15:38:18.098 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:17 smithi164 podman[54892]: 2024-04-26 15:38:17.628510869 +0000 UTC m=+0.418558279 container init b68352a89f2e5137c025a2e2a6556c51a993656435747fb9a79467ccf76f2c57 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0) 2024-04-26T15:38:18.098 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:17 smithi164 podman[54892]: 2024-04-26 15:38:17.631944547 +0000 UTC m=+0.421991965 container start b68352a89f2e5137c025a2e2a6556c51a993656435747fb9a79467ccf76f2c57 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-26T15:38:18.098 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:17 smithi164 ceph-osd[54910]: -- 172.21.15.164:0/3057864731 <== mon.1 v2:172.21.15.164:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55716e625680 con 0x55716e5ef000 2024-04-26T15:38:18.098 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:17 smithi164 bash[54892]: b68352a89f2e5137c025a2e2a6556c51a993656435747fb9a79467ccf76f2c57 2024-04-26T15:38:18.098 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:17 smithi164 systemd[1]: Started Ceph osd.4 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:38:18.597 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:18 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:38:18.175+0000 7f91082557c0 -1 Falling back to public interface 2024-04-26T15:38:18.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 244473 10.28 MB/sec execute 462 sec latency 48.572 ms 2024-04-26T15:38:18.973 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:18 smithi112 ceph-mon[63158]: pgmap v200: 97 pgs: 12 active+undersized, 21 active+undersized+degraded, 8 peering, 56 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.9 MiB/s wr, 545 op/s; 176/1614 objects degraded (10.905%) 2024-04-26T15:38:18.973 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:18 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:18.973 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:18 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:18.973 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:18 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:38:19.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:18 smithi164 ceph-mon[49952]: pgmap v200: 97 pgs: 12 active+undersized, 21 active+undersized+degraded, 8 peering, 56 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.9 MiB/s wr, 545 op/s; 176/1614 objects degraded (10.905%) 2024-04-26T15:38:19.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:18 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:19.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:18 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:19.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:18 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:38:19.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:19 smithi164 ceph-mon[49952]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T15:38:19.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 244730 10.27 MB/sec execute 463 sec latency 69.854 ms 2024-04-26T15:38:19.994 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:19 smithi112 ceph-mon[63158]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T15:38:20.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 245027 10.26 MB/sec execute 464 sec latency 75.281 ms 2024-04-26T15:38:21.014 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:20 smithi112 ceph-mon[63158]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.1 MiB/s wr, 451 op/s; 240/1617 objects degraded (14.842%) 2024-04-26T15:38:21.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:20 smithi164 ceph-mon[49952]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.1 MiB/s wr, 451 op/s; 240/1617 objects degraded (14.842%) 2024-04-26T15:38:21.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 245277 10.25 MB/sec execute 465 sec latency 53.395 ms 2024-04-26T15:38:22.031 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:21 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 255/1686 objects degraded (15.125%), 28 pgs degraded (PG_DEGRADED) 2024-04-26T15:38:22.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:21 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 255/1686 objects degraded (15.125%), 28 pgs degraded (PG_DEGRADED) 2024-04-26T15:38:22.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 245507 10.25 MB/sec execute 466 sec latency 70.441 ms 2024-04-26T15:38:22.941 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:22 smithi112 ceph-mon[63158]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 494 op/s; 255/1686 objects degraded (15.125%) 2024-04-26T15:38:22.941 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:22.941 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:23.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:22 smithi164 ceph-mon[49952]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 494 op/s; 255/1686 objects degraded (15.125%) 2024-04-26T15:38:23.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:23.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:23.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 245773 10.24 MB/sec execute 467 sec latency 57.321 ms 2024-04-26T15:38:24.622 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:24 smithi112 ceph-mon[63158]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.3 MiB/s wr, 444 op/s; 248/1614 objects degraded (15.366%) 2024-04-26T15:38:24.623 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:24.623 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:24.623 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:24.623 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:38:24.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:24 smithi164 ceph-mon[49952]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.3 MiB/s wr, 444 op/s; 248/1614 objects degraded (15.366%) 2024-04-26T15:38:24.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:24.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:24.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:24.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:38:24.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 245964 10.23 MB/sec execute 468 sec latency 92.814 ms 2024-04-26T15:38:25.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:25 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:38:25.418+0000 7f91082557c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-26T15:38:25.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 246275 10.22 MB/sec execute 469 sec latency 87.856 ms 2024-04-26T15:38:26.643 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:26 smithi112 ceph-mon[63158]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.6 MiB/s wr, 497 op/s; 236/1551 objects degraded (15.216%) 2024-04-26T15:38:26.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:26 smithi164 ceph-mon[49952]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.6 MiB/s wr, 497 op/s; 236/1551 objects degraded (15.216%) 2024-04-26T15:38:26.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 246551 10.21 MB/sec execute 470 sec latency 41.779 ms 2024-04-26T15:38:27.347 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:27 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:38:27.132+0000 7f91082557c0 -1 osd.4 114 log_to_monitors true 2024-04-26T15:38:27.614 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:27 smithi112 ceph-mon[63158]: from='osd.4 [v2:172.21.15.164:6800/14702441,v1:172.21.15.164:6801/14702441]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:38:27.615 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:27 smithi112 ceph-mon[63158]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:38:27.615 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:27 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:27.615 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:27 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:27.615 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:27 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:38:27.847 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:38:27 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:38:27.545+0000 7f90f96ad700 -1 osd.4 114 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:38:27.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:27 smithi164 ceph-mon[49952]: from='osd.4 [v2:172.21.15.164:6800/14702441,v1:172.21.15.164:6801/14702441]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:38:27.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:27 smithi164 ceph-mon[49952]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:38:27.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:27 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:27.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:27 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:27.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:27 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:38:27.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 246852 10.21 MB/sec execute 471 sec latency 41.919 ms 2024-04-26T15:38:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 396 op/s; 236/1554 objects degraded (15.187%) 2024-04-26T15:38:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 236/1554 objects degraded (15.187%), 28 pgs degraded (PG_DEGRADED) 2024-04-26T15:38:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:38:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T15:38:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: from='osd.4 [v2:172.21.15.164:6800/14702441,v1:172.21.15.164:6801/14702441]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:38:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: osdmap e117: 8 total, 7 up, 8 in 2024-04-26T15:38:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:38:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:28.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:28.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:38:28.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:38:28.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:38:28.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:38:28.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T15:38:28.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T15:38:28.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:28 smithi112 ceph-mon[63158]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-26T15:38:28.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 396 op/s; 236/1554 objects degraded (15.187%) 2024-04-26T15:38:28.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 236/1554 objects degraded (15.187%), 28 pgs degraded (PG_DEGRADED) 2024-04-26T15:38:28.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:38:28.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T15:38:28.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: from='osd.4 [v2:172.21.15.164:6800/14702441,v1:172.21.15.164:6801/14702441]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:38:28.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: osdmap e117: 8 total, 7 up, 8 in 2024-04-26T15:38:28.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:38:28.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:28.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:28.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:38:28.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:38:28.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:38:28.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:38:28.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T15:38:28.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T15:38:28.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:28 smithi164 ceph-mon[49952]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-26T15:38:28.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 247158 10.21 MB/sec execute 472 sec latency 334.749 ms 2024-04-26T15:38:29.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:29 smithi164 ceph-mon[49952]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:38:29.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:29 smithi164 ceph-mon[49952]: osd.4 [v2:172.21.15.164:6800/14702441,v1:172.21.15.164:6801/14702441] boot 2024-04-26T15:38:29.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:29 smithi164 ceph-mon[49952]: osdmap e118: 8 total, 8 up, 8 in 2024-04-26T15:38:29.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:29 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:38:29.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:29 smithi164 ceph-mon[49952]: osdmap e119: 8 total, 8 up, 8 in 2024-04-26T15:38:29.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 247158 10.19 MB/sec execute 473 sec latency 1334.795 ms 2024-04-26T15:38:29.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:29 smithi112 ceph-mon[63158]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:38:29.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:29 smithi112 ceph-mon[63158]: osd.4 [v2:172.21.15.164:6800/14702441,v1:172.21.15.164:6801/14702441] boot 2024-04-26T15:38:29.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:29 smithi112 ceph-mon[63158]: osdmap e118: 8 total, 8 up, 8 in 2024-04-26T15:38:29.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:29 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:38:29.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:29 smithi112 ceph-mon[63158]: osdmap e119: 8 total, 8 up, 8 in 2024-04-26T15:38:30.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:30 smithi112 ceph-mon[63158]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.9 MiB/s wr, 269 op/s; 247/1605 objects degraded (15.389%) 2024-04-26T15:38:30.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:30 smithi112 ceph-mon[63158]: osdmap e120: 8 total, 8 up, 8 in 2024-04-26T15:38:30.839 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:38:30.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:30 smithi164 ceph-mon[49952]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.9 MiB/s wr, 269 op/s; 247/1605 objects degraded (15.389%) 2024-04-26T15:38:30.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:30 smithi164 ceph-mon[49952]: osdmap e120: 8 total, 8 up, 8 in 2024-04-26T15:38:30.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 247258 10.18 MB/sec execute 474 sec latency 1923.860 ms 2024-04-26T15:38:31.732 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:38:31.732 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (7m) 47s ago 16m 25.1M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:38:31.732 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (4m) 47s ago 16m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:38:31.732 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (3m) 9s ago 15m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:38:31.732 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (7m) 47s ago 16m 54.1M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:38:31.732 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (10m) 47s ago 10m 38.3M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:38:31.732 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (10m) 9s ago 10m 14.7M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:38:31.732 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (6m) 47s ago 17m 522M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:38:31.732 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (5m) 9s ago 15m 468M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:38:31.732 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (5m) 47s ago 17m 52.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:38:31.732 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (5m) 9s ago 15m 41.4M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (10m) 47s ago 10m 266M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (10m) 9s ago 10m 90.2M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (7m) 47s ago 16m 17.5M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (6m) 9s ago 15m 18.6M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (3m) 47s ago 15m 353M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (2m) 47s ago 14m 291M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (105s) 47s ago 14m 213M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (58s) 47s ago 13m 14.9M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd 27fefb9c15b3 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (14s) 9s ago 13m 15.1M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd b68352a89f2e 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (12m) 9s ago 12m 569M 2704M 17.2.0 e1d6a67b021e 58cff7d2fc37 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (12m) 9s ago 12m 498M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (11m) 9s ago 11m 770M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:38:31.733 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (6m) 47s ago 15m 73.2M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:38:31.734 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:31 smithi112 ceph-mon[63158]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:38:31.734 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:31 smithi112 ceph-mon[63158]: osdmap e121: 8 total, 8 up, 8 in 2024-04-26T15:38:31.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:31 smithi164 ceph-mon[49952]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:38:31.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:31 smithi164 ceph-mon[49952]: osdmap e121: 8 total, 8 up, 8 in 2024-04-26T15:38:31.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 247585 10.18 MB/sec execute 475 sec latency 45.548 ms 2024-04-26T15:38:32.448 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:38:32.448 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:38:32.448 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:38:32.448 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 5 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-26T15:38:32.449 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 9 2024-04-26T15:38:32.450 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:38:32.450 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:38:32.742 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:32 smithi112 ceph-mon[63158]: pgmap v211: 97 pgs: 1 active+recovering+degraded, 4 active+recovery_wait+degraded, 6 remapped+peering, 2 peering, 12 active+undersized, 12 active+undersized+degraded, 60 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 390 op/s; 160/1611 objects degraded (9.932%); 0 B/s, 3 objects/s recovering 2024-04-26T15:38:32.743 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:32 smithi112 ceph-mon[63158]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:38:32.743 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:32 smithi112 ceph-mon[63158]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:38:32.743 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:32 smithi112 ceph-mon[63158]: osdmap e122: 8 total, 8 up, 8 in 2024-04-26T15:38:32.743 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:32 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/3320646080' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:38:32.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:32 smithi164 ceph-mon[49952]: pgmap v211: 97 pgs: 1 active+recovering+degraded, 4 active+recovery_wait+degraded, 6 remapped+peering, 2 peering, 12 active+undersized, 12 active+undersized+degraded, 60 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 390 op/s; 160/1611 objects degraded (9.932%); 0 B/s, 3 objects/s recovering 2024-04-26T15:38:32.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:32 smithi164 ceph-mon[49952]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:38:32.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:32 smithi164 ceph-mon[49952]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:38:32.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:32 smithi164 ceph-mon[49952]: osdmap e122: 8 total, 8 up, 8 in 2024-04-26T15:38:32.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:32 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/3320646080' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:38:32.870 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:38:32.871 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:38:32.871 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:38:32.871 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:38:32.871 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:38:32.871 INFO:teuthology.orchestra.run.smithi112.stdout: "crash", 2024-04-26T15:38:32.871 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:38:32.871 INFO:teuthology.orchestra.run.smithi112.stdout: "mon" 2024-04-26T15:38:32.871 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:38:32.871 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "11/23 daemons upgraded", 2024-04-26T15:38:32.871 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T15:38:32.871 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:38:32.871 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:38:32.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 247848 10.17 MB/sec execute 476 sec latency 499.784 ms 2024-04-26T15:38:33.292 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_WARN Degraded data redundancy: 160/1611 objects degraded (9.932%), 17 pgs degraded 2024-04-26T15:38:33.292 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 160/1611 objects degraded (9.932%), 17 pgs degraded 2024-04-26T15:38:33.292 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-04-26T15:38:33.292 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-04-26T15:38:33.292 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-04-26T15:38:33.292 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-04-26T15:38:33.292 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1f is active+undersized+degraded, acting [0,7] 2024-04-26T15:38:33.292 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-04-26T15:38:33.292 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-04-26T15:38:33.292 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-04-26T15:38:33.293 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.a is active+recovery_wait+degraded, acting [6,4,1] 2024-04-26T15:38:33.293 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.f is active+recovery_wait+degraded, acting [7,4,0] 2024-04-26T15:38:33.293 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.13 is active+recovery_wait+degraded, acting [7,4,2] 2024-04-26T15:38:33.293 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-04-26T15:38:33.293 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.15 is active+recovery_wait+degraded, acting [7,3,4] 2024-04-26T15:38:33.293 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-04-26T15:38:33.293 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-04-26T15:38:33.293 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-04-26T15:38:33.293 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1c is active+recovering+degraded, acting [5,4,1] 2024-04-26T15:38:33.806 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:33 smithi112 ceph-mon[63158]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:38:33.806 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:33 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/2844972222' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:38:33.806 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:33 smithi112 ceph-mon[63158]: osdmap e123: 8 total, 8 up, 8 in 2024-04-26T15:38:33.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:33 smithi164 ceph-mon[49952]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:38:33.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:33 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/2844972222' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:38:33.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:33 smithi164 ceph-mon[49952]: osdmap e123: 8 total, 8 up, 8 in 2024-04-26T15:38:33.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 247959 10.15 MB/sec execute 477 sec latency 1042.232 ms 2024-04-26T15:38:34.576 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:34 smithi112 ceph-mon[63158]: pgmap v214: 97 pgs: 1 active+recovering+degraded, 4 active+recovery_wait+degraded, 5 remapped+peering, 4 peering, 83 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 409 op/s; 51/1608 objects degraded (3.172%); 784 KiB/s, 62 keys/s, 40 objects/s recovering 2024-04-26T15:38:34.576 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:34 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 51/1608 objects degraded (3.172%), 5 pgs degraded (PG_DEGRADED) 2024-04-26T15:38:34.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:34 smithi164 ceph-mon[49952]: pgmap v214: 97 pgs: 1 active+recovering+degraded, 4 active+recovery_wait+degraded, 5 remapped+peering, 4 peering, 83 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 409 op/s; 51/1608 objects degraded (3.172%); 784 KiB/s, 62 keys/s, 40 objects/s recovering 2024-04-26T15:38:34.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:34 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 51/1608 objects degraded (3.172%), 5 pgs degraded (PG_DEGRADED) 2024-04-26T15:38:34.871 INFO:teuthology.orchestra.run.smithi112.stdout: 5 248201 10.14 MB/sec execute 478 sec latency 66.288 ms 2024-04-26T15:38:35.867 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:35 smithi112 ceph-mon[63158]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 51/1608 objects degraded (3.172%), 5 pgs degraded) 2024-04-26T15:38:35.867 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:35 smithi112 ceph-mon[63158]: Cluster is now healthy 2024-04-26T15:38:35.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 248492 10.13 MB/sec execute 479 sec latency 56.651 ms 2024-04-26T15:38:36.042 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:35 smithi164 ceph-mon[49952]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 51/1608 objects degraded (3.172%), 5 pgs degraded) 2024-04-26T15:38:36.042 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:35 smithi164 ceph-mon[49952]: Cluster is now healthy 2024-04-26T15:38:36.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:36 smithi112 ceph-mon[63158]: pgmap v216: 97 pgs: 2 peering, 95 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.2 MiB/s wr, 348 op/s; 2.3 MiB/s, 52 keys/s, 59 objects/s recovering 2024-04-26T15:38:36.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 248751 10.12 MB/sec execute 480 sec latency 66.494 ms 2024-04-26T15:38:37.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:36 smithi164 ceph-mon[49952]: pgmap v216: 97 pgs: 2 peering, 95 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.2 MiB/s wr, 348 op/s; 2.3 MiB/s, 52 keys/s, 59 objects/s recovering 2024-04-26T15:38:37.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 249051 10.12 MB/sec execute 481 sec latency 72.596 ms 2024-04-26T15:38:38.747 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:38 smithi112 ceph-mon[63158]: pgmap v217: 97 pgs: 2 peering, 95 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.8 MiB/s wr, 282 op/s; 1.8 MiB/s, 41 keys/s, 46 objects/s recovering 2024-04-26T15:38:38.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 249307 10.11 MB/sec execute 482 sec latency 43.422 ms 2024-04-26T15:38:39.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:38 smithi164 ceph-mon[49952]: pgmap v217: 97 pgs: 2 peering, 95 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.8 MiB/s wr, 282 op/s; 1.8 MiB/s, 41 keys/s, 46 objects/s recovering 2024-04-26T15:38:39.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 249564 10.11 MB/sec execute 483 sec latency 73.850 ms 2024-04-26T15:38:40.327 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:40 smithi112 ceph-mon[63158]: pgmap v218: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.5 MiB/s wr, 245 op/s; 1.4 MiB/s, 34 objects/s recovering 2024-04-26T15:38:40.327 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:40.327 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:38:40.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:40 smithi164 ceph-mon[49952]: pgmap v218: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.5 MiB/s wr, 245 op/s; 1.4 MiB/s, 34 objects/s recovering 2024-04-26T15:38:40.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:40.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:38:40.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 249873 10.11 MB/sec execute 484 sec latency 67.590 ms 2024-04-26T15:38:41.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 250108 10.09 MB/sec execute 485 sec latency 55.360 ms 2024-04-26T15:38:42.115 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:42 smithi112 ceph-mon[63158]: pgmap v219: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.7 MiB/s wr, 413 op/s; 1.2 MiB/s, 30 objects/s recovering 2024-04-26T15:38:42.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:42 smithi164 ceph-mon[49952]: pgmap v219: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.7 MiB/s wr, 413 op/s; 1.2 MiB/s, 30 objects/s recovering 2024-04-26T15:38:42.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 250303 10.08 MB/sec execute 486 sec latency 52.126 ms 2024-04-26T15:38:43.159 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:43 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T15:38:43.159 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:43 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T15:38:43.159 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:43 smithi112 ceph-mon[63158]: Upgrade: osd.5 is safe to restart 2024-04-26T15:38:43.377 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:43 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T15:38:43.378 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:43 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T15:38:43.378 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:43 smithi164 ceph-mon[49952]: Upgrade: osd.5 is safe to restart 2024-04-26T15:38:43.874 INFO:teuthology.orchestra.run.smithi112.stdout: 5 250593 10.09 MB/sec execute 487 sec latency 53.298 ms 2024-04-26T15:38:44.170 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:44 smithi112 ceph-mon[63158]: pgmap v220: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.5 MiB/s wr, 368 op/s; 794 KiB/s, 13 objects/s recovering 2024-04-26T15:38:44.383 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:44 smithi164 ceph-mon[49952]: pgmap v220: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.5 MiB/s wr, 368 op/s; 794 KiB/s, 13 objects/s recovering 2024-04-26T15:38:44.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 250994 10.09 MB/sec execute 488 sec latency 43.252 ms 2024-04-26T15:38:45.181 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:45.182 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:45 smithi112 ceph-mon[63158]: Upgrade: Updating osd.5 2024-04-26T15:38:45.182 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:45.182 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T15:38:45.182 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:38:45.182 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:45 smithi112 ceph-mon[63158]: Deploying daemon osd.5 on smithi164 2024-04-26T15:38:45.417 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:45.417 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:45 smithi164 ceph-mon[49952]: Upgrade: Updating osd.5 2024-04-26T15:38:45.417 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:45.417 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T15:38:45.417 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:38:45.417 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:45 smithi164 ceph-mon[49952]: Deploying daemon osd.5 on smithi164 2024-04-26T15:38:45.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 251494 10.10 MB/sec execute 489 sec latency 56.323 ms 2024-04-26T15:38:46.581 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:46 smithi164 ceph-mon[49952]: pgmap v221: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.0 MiB/s wr, 392 op/s; 677 KiB/s, 11 objects/s recovering 2024-04-26T15:38:46.703 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:46 smithi112 ceph-mon[63158]: pgmap v221: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.0 MiB/s wr, 392 op/s; 677 KiB/s, 11 objects/s recovering 2024-04-26T15:38:46.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 251869 10.09 MB/sec execute 490 sec latency 57.869 ms 2024-04-26T15:38:47.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 252205 10.09 MB/sec execute 491 sec latency 46.313 ms 2024-04-26T15:38:48.525 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:48 smithi164 systemd[1]: Stopping Ceph osd.5 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:38:48.740 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:48 smithi112 ceph-mon[63158]: pgmap v222: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.8 MiB/s wr, 353 op/s 2024-04-26T15:38:48.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:48 smithi164 ceph-mon[49952]: pgmap v222: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.8 MiB/s wr, 353 op/s 2024-04-26T15:38:48.847 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:48 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[31286]: 2024-04-26T15:38:48.624+0000 7f1d69380700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:38:48.848 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:48 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[31286]: 2024-04-26T15:38:48.624+0000 7f1d69380700 -1 osd.5 123 *** Got signal Terminated *** 2024-04-26T15:38:48.848 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:48 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[31286]: 2024-04-26T15:38:48.624+0000 7f1d69380700 -1 osd.5 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:38:48.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 252542 10.09 MB/sec execute 492 sec latency 51.948 ms 2024-04-26T15:38:49.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:49 smithi164 ceph-mon[49952]: osd.5 marked itself down and dead 2024-04-26T15:38:49.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 252770 10.08 MB/sec execute 493 sec latency 352.608 ms 2024-04-26T15:38:49.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:49 smithi112 ceph-mon[63158]: osd.5 marked itself down and dead 2024-04-26T15:38:50.473 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:50 smithi164 podman[56724]: 2024-04-26 15:38:50.186415229 +0000 UTC m=+1.700874555 container died 58cff7d2fc371caa3e79015c1155c2166eb546589ca0917529cd5592ec426c45 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, architecture=x86_64, com.redhat.component=centos-stream-container, release=754, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, ceph=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git) 2024-04-26T15:38:50.787 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:50 smithi112 ceph-mon[63158]: pgmap v223: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.1 MiB/s wr, 375 op/s 2024-04-26T15:38:50.787 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:50 smithi112 ceph-mon[63158]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:38:50.788 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:50 smithi112 ceph-mon[63158]: osdmap e124: 8 total, 7 up, 8 in 2024-04-26T15:38:50.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:50 smithi164 ceph-mon[49952]: pgmap v223: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.1 MiB/s wr, 375 op/s 2024-04-26T15:38:50.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:50 smithi164 ceph-mon[49952]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:38:50.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:50 smithi164 ceph-mon[49952]: osdmap e124: 8 total, 7 up, 8 in 2024-04-26T15:38:50.848 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:50 smithi164 podman[56724]: 2024-04-26 15:38:50.556829799 +0000 UTC m=+2.071289116 container cleanup 58cff7d2fc371caa3e79015c1155c2166eb546589ca0917529cd5592ec426c45 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5, vcs-type=git, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, ceph=True, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, release=754, io.openshift.expose-services=, architecture=x86_64, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, version=8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-26T15:38:50.848 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:50 smithi164 bash[56724]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5 2024-04-26T15:38:50.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 252868 10.07 MB/sec execute 494 sec latency 1031.422 ms 2024-04-26T15:38:51.227 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:50 smithi164 podman[56743]: 2024-04-26 15:38:50.858371704 +0000 UTC m=+0.667771921 container remove 58cff7d2fc371caa3e79015c1155c2166eb546589ca0917529cd5592ec426c45 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , release=754, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., name=centos-stream, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, ceph=True, io.openshift.tags=base centos centos-stream) 2024-04-26T15:38:51.521 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:51 smithi164 podman[56833]: 2024-04-26 15:38:51.131049047 +0000 UTC m=+0.021941647 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:38:51.521 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:51 smithi164 podman[56833]: 2024-04-26 15:38:51.259537851 +0000 UTC m=+0.150430438 container create af759169759f9e5a0eddd5f4a907855191dbe3b786a0a571f536656324e75526 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424) 2024-04-26T15:38:51.521 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:51 smithi164 podman[56833]: 2024-04-26 15:38:51.52226582 +0000 UTC m=+0.413158411 container init af759169759f9e5a0eddd5f4a907855191dbe3b786a0a571f536656324e75526 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-26T15:38:51.553 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:51 smithi112 ceph-mon[63158]: osdmap e125: 8 total, 7 up, 8 in 2024-04-26T15:38:51.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:51 smithi164 ceph-mon[49952]: osdmap e125: 8 total, 7 up, 8 in 2024-04-26T15:38:51.848 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:51 smithi164 podman[56833]: 2024-04-26 15:38:51.525615306 +0000 UTC m=+0.416507896 container start af759169759f9e5a0eddd5f4a907855191dbe3b786a0a571f536656324e75526 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-26T15:38:51.848 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:51 smithi164 podman[56833]: 2024-04-26 15:38:51.621152919 +0000 UTC m=+0.512045535 container attach af759169759f9e5a0eddd5f4a907855191dbe3b786a0a571f536656324e75526 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:38:51.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 253134 10.06 MB/sec execute 495 sec latency 75.844 ms 2024-04-26T15:38:52.332 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:51 smithi164 podman[56833]: 2024-04-26 15:38:51.874628058 +0000 UTC m=+0.765520640 container died af759169759f9e5a0eddd5f4a907855191dbe3b786a0a571f536656324e75526 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-26T15:38:52.573 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:52 smithi112 ceph-mon[63158]: pgmap v226: 97 pgs: 9 peering, 88 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 432 op/s 2024-04-26T15:38:52.573 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:52 smithi112 ceph-mon[63158]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T15:38:52.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:52 smithi164 ceph-mon[49952]: pgmap v226: 97 pgs: 9 peering, 88 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 432 op/s 2024-04-26T15:38:52.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:52 smithi164 ceph-mon[49952]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T15:38:52.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 253500 10.06 MB/sec execute 496 sec latency 51.916 ms 2024-04-26T15:38:53.094 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:52 smithi164 podman[56859]: 2024-04-26 15:38:52.625825258 +0000 UTC m=+0.746006984 container remove af759169759f9e5a0eddd5f4a907855191dbe3b786a0a571f536656324e75526 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-26T15:38:53.094 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:52 smithi164 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.5.service: Deactivated successfully. 2024-04-26T15:38:53.094 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:52 smithi164 systemd[1]: Stopped Ceph osd.5 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:38:53.094 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:52 smithi164 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.5.service: Consumed 5min 14.648s CPU time. 2024-04-26T15:38:53.094 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:52 smithi164 systemd[1]: Starting Ceph osd.5 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:38:53.348 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 podman[56958]: 2024-04-26 15:38:52.997651998 +0000 UTC m=+0.019879763 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:38:53.348 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 podman[56958]: 2024-04-26 15:38:53.127521473 +0000 UTC m=+0.149749224 container create 8f2143f180eae634e7525499ac15ca1e33ff66688a343e55c3ca1b33eb97fc96 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:38:53.836 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:53 smithi112 ceph-mon[63158]: Health check failed: Degraded data redundancy: 74/1617 objects degraded (4.576%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T15:38:53.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:53 smithi164 ceph-mon[49952]: Health check failed: Degraded data redundancy: 74/1617 objects degraded (4.576%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T15:38:53.847 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 podman[56958]: 2024-04-26 15:38:53.39768692 +0000 UTC m=+0.419914675 container init 8f2143f180eae634e7525499ac15ca1e33ff66688a343e55c3ca1b33eb97fc96 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:38:53.848 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 podman[56958]: 2024-04-26 15:38:53.403803931 +0000 UTC m=+0.426031683 container start 8f2143f180eae634e7525499ac15ca1e33ff66688a343e55c3ca1b33eb97fc96 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-26T15:38:53.848 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 podman[56958]: 2024-04-26 15:38:53.505601764 +0000 UTC m=+0.527829529 container attach 8f2143f180eae634e7525499ac15ca1e33ff66688a343e55c3ca1b33eb97fc96 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-26T15:38:53.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 253749 10.05 MB/sec execute 497 sec latency 39.926 ms 2024-04-26T15:38:54.326 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate[56973]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:38:54.326 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate[56973]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:38:54.326 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 bash[56958]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:38:54.326 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 bash[56958]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:38:54.326 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 bash[56958]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-26T15:38:54.326 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 bash[56958]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T15:38:54.326 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 bash[56958]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T15:38:54.326 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 bash[56958]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T15:38:54.327 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 bash[56958]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-26T15:38:54.327 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 bash[56958]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-26T15:38:54.327 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 bash[56958]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-26T15:38:54.327 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate[56973]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-26T15:38:54.327 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate[56973]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T15:38:54.327 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate[56973]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T15:38:54.327 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate[56973]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T15:38:54.327 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate[56973]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-26T15:38:54.327 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate[56973]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-26T15:38:54.327 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate[56973]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-26T15:38:54.327 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:53 smithi164 podman[56958]: 2024-04-26 15:38:53.969099043 +0000 UTC m=+0.991326799 container died 8f2143f180eae634e7525499ac15ca1e33ff66688a343e55c3ca1b33eb97fc96 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:38:54.615 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:54 smithi112 ceph-mon[63158]: pgmap v227: 97 pgs: 5 active+undersized, 8 active+undersized+degraded, 9 peering, 75 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.5 MiB/s wr, 486 op/s; 74/1617 objects degraded (4.576%) 2024-04-26T15:38:54.615 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:54.616 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:38:54.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:54 smithi164 ceph-mon[49952]: pgmap v227: 97 pgs: 5 active+undersized, 8 active+undersized+degraded, 9 peering, 75 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.5 MiB/s wr, 486 op/s; 74/1617 objects degraded (4.576%) 2024-04-26T15:38:54.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:54.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:38:54.848 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:54 smithi164 podman[57111]: 2024-04-26 15:38:54.69443955 +0000 UTC m=+0.719445783 container remove 8f2143f180eae634e7525499ac15ca1e33ff66688a343e55c3ca1b33eb97fc96 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-26T15:38:54.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 253960 10.04 MB/sec execute 498 sec latency 66.630 ms 2024-04-26T15:38:55.240 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:54 smithi164 podman[57177]: 2024-04-26 15:38:54.81881845 +0000 UTC m=+0.020971508 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:38:55.240 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:54 smithi164 podman[57177]: 2024-04-26 15:38:54.937157279 +0000 UTC m=+0.139310305 container create 61be8ee4a0ab84c7830590a8eafe214d30dcaf681afa960b732f5b0340d8ccf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-26T15:38:55.597 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:55 smithi164 podman[57177]: 2024-04-26 15:38:55.240027714 +0000 UTC m=+0.442180740 container init 61be8ee4a0ab84c7830590a8eafe214d30dcaf681afa960b732f5b0340d8ccf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-26T15:38:55.598 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:55 smithi164 podman[57177]: 2024-04-26 15:38:55.245292578 +0000 UTC m=+0.447445613 container start 61be8ee4a0ab84c7830590a8eafe214d30dcaf681afa960b732f5b0340d8ccf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-26T15:38:55.598 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:55 smithi164 ceph-osd[57197]: -- 172.21.15.164:0/314938457 <== mon.1 v2:172.21.15.164:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e6295c7680 con 0x55e629590c00 2024-04-26T15:38:55.598 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:55 smithi164 bash[57177]: 61be8ee4a0ab84c7830590a8eafe214d30dcaf681afa960b732f5b0340d8ccf9 2024-04-26T15:38:55.598 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:55 smithi164 systemd[1]: Started Ceph osd.5 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:38:55.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 254251 10.04 MB/sec execute 499 sec latency 48.859 ms 2024-04-26T15:38:56.097 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:38:55 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:38:55.800+0000 7f8689ba97c0 -1 Falling back to public interface 2024-04-26T15:38:56.631 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:56 smithi112 ceph-mon[63158]: pgmap v228: 97 pgs: 8 active+undersized, 21 active+undersized+degraded, 2 peering, 66 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.3 MiB/s wr, 405 op/s; 190/1620 objects degraded (11.728%) 2024-04-26T15:38:56.631 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:56 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:56.631 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:56 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:56.631 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:56 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:38:56.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:56 smithi164 ceph-mon[49952]: pgmap v228: 97 pgs: 8 active+undersized, 21 active+undersized+degraded, 2 peering, 66 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.3 MiB/s wr, 405 op/s; 190/1620 objects degraded (11.728%) 2024-04-26T15:38:56.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:56 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:56.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:56 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:38:56.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:56 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:38:56.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 254554 10.04 MB/sec execute 500 sec latency 53.637 ms 2024-04-26T15:38:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:57 smithi112 ceph-mon[63158]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T15:38:57.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:57 smithi164 ceph-mon[49952]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T15:38:57.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 254933 10.04 MB/sec execute 501 sec latency 72.146 ms 2024-04-26T15:38:58.699 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:58 smithi112 ceph-mon[63158]: pgmap v229: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.6 MiB/s wr, 513 op/s; 198/1644 objects degraded (12.044%) 2024-04-26T15:38:58.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:58 smithi164 ceph-mon[49952]: pgmap v229: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.6 MiB/s wr, 513 op/s; 198/1644 objects degraded (12.044%) 2024-04-26T15:38:58.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 255230 10.03 MB/sec execute 502 sec latency 55.497 ms 2024-04-26T15:38:59.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:38:59 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 198/1644 objects degraded (12.044%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T15:38:59.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:38:59 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 198/1644 objects degraded (12.044%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T15:38:59.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 255612 10.03 MB/sec execute 503 sec latency 47.354 ms 2024-04-26T15:39:00.745 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:00 smithi112 ceph-mon[63158]: pgmap v230: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 439 op/s; 194/1659 objects degraded (11.694%) 2024-04-26T15:39:00.746 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:00 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:00.746 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:00 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:00.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:00 smithi164 ceph-mon[49952]: pgmap v230: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 439 op/s; 194/1659 objects degraded (11.694%) 2024-04-26T15:39:00.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:00 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:00.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:00 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:00.872 INFO:teuthology.orchestra.run.smithi112.stdout: 5 255980 10.03 MB/sec execute 504 sec latency 51.014 ms 2024-04-26T15:39:01.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 256366 10.03 MB/sec execute 505 sec latency 37.029 ms 2024-04-26T15:39:02.013 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:01 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:02.013 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:01 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:02.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:01 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:02.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:01 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:02.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 256752 10.03 MB/sec execute 506 sec latency 55.736 ms 2024-04-26T15:39:02.984 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:02 smithi164 ceph-mon[49952]: pgmap v231: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 432 op/s; 197/1671 objects degraded (11.789%) 2024-04-26T15:39:03.040 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:02 smithi112 ceph-mon[63158]: pgmap v231: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 432 op/s; 197/1671 objects degraded (11.789%) 2024-04-26T15:39:03.807 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:39:03.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 257166 10.03 MB/sec execute 507 sec latency 41.350 ms 2024-04-26T15:39:04.569 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:39:04 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:39:04.246+0000 7f8689ba97c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-26T15:39:04.749 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (7m) 80s ago 17m 25.1M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (4m) 80s ago 17m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (4m) 5s ago 16m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (7m) 80s ago 16m 54.1M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (11m) 80s ago 10m 38.3M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (10m) 5s ago 10m 14.8M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (7m) 80s ago 18m 522M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (6m) 5s ago 16m 468M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (6m) 80s ago 18m 52.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (5m) 5s ago 16m 45.5M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (10m) 80s ago 10m 266M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (10m) 5s ago 10m 90.2M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:39:04.750 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (7m) 80s ago 16m 17.5M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:39:04.751 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (7m) 5s ago 16m 18.1M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:39:04.751 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (3m) 80s ago 15m 353M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:39:04.751 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (3m) 80s ago 15m 291M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:39:04.751 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (2m) 80s ago 14m 213M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:39:04.751 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (91s) 80s ago 14m 14.9M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd 27fefb9c15b3 2024-04-26T15:39:04.751 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (47s) 5s ago 13m 266M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd b68352a89f2e 2024-04-26T15:39:04.751 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (9s) 5s ago 13m 14.7M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 61be8ee4a0ab 2024-04-26T15:39:04.751 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (12m) 5s ago 12m 533M 2704M 17.2.0 e1d6a67b021e d77044ffb3d6 2024-04-26T15:39:04.751 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (12m) 5s ago 12m 804M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:39:04.751 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (6m) 80s ago 15m 73.2M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:39:04.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 257449 10.02 MB/sec execute 508 sec latency 54.516 ms 2024-04-26T15:39:05.056 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:04 smithi112 ceph-mon[63158]: pgmap v232: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.9 MiB/s wr, 548 op/s; 192/1650 objects degraded (11.636%) 2024-04-26T15:39:05.057 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:04 smithi112 ceph-mon[63158]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:05.057 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:04 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 192/1650 objects degraded (11.636%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:05.338 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:39:05.338 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:39:05.338 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:39:05.338 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:39:05.338 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:39:05.338 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:39:05.338 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:39:05.339 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:39:05.339 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-26T15:39:05.339 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 5 2024-04-26T15:39:05.339 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:39:05.339 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:39:05.339 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:39:05.339 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:39:05.339 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:39:05.339 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-26T15:39:05.339 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 9 2024-04-26T15:39:05.339 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:39:05.339 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:39:05.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:04 smithi164 ceph-mon[49952]: pgmap v232: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.9 MiB/s wr, 548 op/s; 192/1650 objects degraded (11.636%) 2024-04-26T15:39:05.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:04 smithi164 ceph-mon[49952]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:05.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:04 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 192/1650 objects degraded (11.636%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:05.814 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:39:05.815 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:39:05.815 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:39:05.815 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:39:05.815 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:39:05.815 INFO:teuthology.orchestra.run.smithi112.stdout: "crash", 2024-04-26T15:39:05.815 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:39:05.815 INFO:teuthology.orchestra.run.smithi112.stdout: "mon" 2024-04-26T15:39:05.815 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:39:05.815 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "12/23 daemons upgraded", 2024-04-26T15:39:05.815 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T15:39:05.815 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:39:05.815 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:39:05.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 257763 10.03 MB/sec execute 509 sec latency 38.603 ms 2024-04-26T15:39:06.084 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:06.084 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:06.084 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:06.084 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:06.084 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:39:06.084 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:39:06.084 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:06.084 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:06.085 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/3782900898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:06.085 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:39:06.085 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:06.085 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:06.085 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:06.085 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T15:39:06.085 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='osd.5 [v2:172.21.15.164:6808/3210627669,v1:172.21.15.164:6809/3210627669]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:39:06.085 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:05 smithi112 ceph-mon[63158]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:39:06.097 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:39:05.762+0000 7f8689ba97c0 -1 osd.5 123 log_to_monitors true 2024-04-26T15:39:06.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/3782900898' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T15:39:06.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='osd.5 [v2:172.21.15.164:6808/3210627669,v1:172.21.15.164:6809/3210627669]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:39:06.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:05 smithi164 ceph-mon[49952]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:39:06.457 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 183/1650 objects degraded (11.091%), 23 pgs degraded 2024-04-26T15:39:06.457 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: osd.5 (root=default,host=smithi164) is down 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 183/1650 objects degraded (11.091%), 23 pgs degraded 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-04-26T15:39:06.458 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-04-26T15:39:06.459 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-04-26T15:39:06.459 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-04-26T15:39:06.459 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-04-26T15:39:06.459 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-04-26T15:39:06.459 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-04-26T15:39:06.459 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-04-26T15:39:06.459 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-04-26T15:39:06.459 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-04-26T15:39:06.459 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-04-26T15:39:06.459 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-04-26T15:39:06.459 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-04-26T15:39:06.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 258028 10.03 MB/sec execute 510 sec latency 163.184 ms 2024-04-26T15:39:06.977 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:39:06 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:39:06.690+0000 7f8676ff9700 -1 osd.5 123 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:39:07.026 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:06 smithi112 ceph-mon[63158]: pgmap v233: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.8 MiB/s wr, 474 op/s; 183/1650 objects degraded (11.091%) 2024-04-26T15:39:07.026 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:06 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T15:39:07.026 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:06 smithi112 ceph-mon[63158]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-26T15:39:07.026 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:06 smithi112 ceph-mon[63158]: from='client.44289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:07.026 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:06 smithi112 ceph-mon[63158]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T15:39:07.026 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:06 smithi112 ceph-mon[63158]: osdmap e126: 8 total, 7 up, 8 in 2024-04-26T15:39:07.026 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:06 smithi112 ceph-mon[63158]: from='osd.5 [v2:172.21.15.164:6808/3210627669,v1:172.21.15.164:6809/3210627669]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:39:07.026 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:06 smithi112 ceph-mon[63158]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:39:07.027 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:06 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/3889468754' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:39:07.027 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:06 smithi112 ceph-mon[63158]: from='osd.5 ' entity='osd.5' 2024-04-26T15:39:07.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:06 smithi164 ceph-mon[49952]: pgmap v233: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.8 MiB/s wr, 474 op/s; 183/1650 objects degraded (11.091%) 2024-04-26T15:39:07.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:06 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T15:39:07.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:06 smithi164 ceph-mon[49952]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-26T15:39:07.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:06 smithi164 ceph-mon[49952]: from='client.44289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:07.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:06 smithi164 ceph-mon[49952]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T15:39:07.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:06 smithi164 ceph-mon[49952]: osdmap e126: 8 total, 7 up, 8 in 2024-04-26T15:39:07.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:06 smithi164 ceph-mon[49952]: from='osd.5 [v2:172.21.15.164:6808/3210627669,v1:172.21.15.164:6809/3210627669]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:39:07.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:06 smithi164 ceph-mon[49952]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:39:07.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:06 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/3889468754' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:39:07.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:06 smithi164 ceph-mon[49952]: from='osd.5 ' entity='osd.5' 2024-04-26T15:39:07.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 258337 10.02 MB/sec execute 511 sec latency 198.904 ms 2024-04-26T15:39:08.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:07 smithi164 ceph-mon[49952]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:39:08.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:07 smithi164 ceph-mon[49952]: osd.5 [v2:172.21.15.164:6808/3210627669,v1:172.21.15.164:6809/3210627669] boot 2024-04-26T15:39:08.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:07 smithi164 ceph-mon[49952]: osdmap e127: 8 total, 8 up, 8 in 2024-04-26T15:39:08.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:07 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:39:08.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:07 smithi112 ceph-mon[63158]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:39:08.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:07 smithi112 ceph-mon[63158]: osd.5 [v2:172.21.15.164:6808/3210627669,v1:172.21.15.164:6809/3210627669] boot 2024-04-26T15:39:08.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:07 smithi112 ceph-mon[63158]: osdmap e127: 8 total, 8 up, 8 in 2024-04-26T15:39:08.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:07 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:39:08.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 258337 10.00 MB/sec execute 512 sec latency 1198.956 ms 2024-04-26T15:39:09.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:08 smithi164 ceph-mon[49952]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 442 op/s; 188/1668 objects degraded (11.271%) 2024-04-26T15:39:09.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:08 smithi164 ceph-mon[49952]: osdmap e128: 8 total, 8 up, 8 in 2024-04-26T15:39:09.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:08 smithi112 ceph-mon[63158]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 442 op/s; 188/1668 objects degraded (11.271%) 2024-04-26T15:39:09.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:08 smithi112 ceph-mon[63158]: osdmap e128: 8 total, 8 up, 8 in 2024-04-26T15:39:09.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 258338 9.99 MB/sec execute 513 sec latency 2191.831 ms 2024-04-26T15:39:10.123 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:10 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:10.123 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:10 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:39:10.123 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:10 smithi112 ceph-mon[63158]: pgmap v238: 97 pgs: 3 peering, 7 active+undersized, 21 active+undersized+degraded, 66 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.5 MiB/s wr, 295 op/s; 161/1656 objects degraded (9.722%) 2024-04-26T15:39:10.123 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:10 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 188/1668 objects degraded (11.271%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:10.123 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:10 smithi112 ceph-mon[63158]: osdmap e129: 8 total, 8 up, 8 in 2024-04-26T15:39:10.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:10 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:10.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:10 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:39:10.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:10 smithi164 ceph-mon[49952]: pgmap v238: 97 pgs: 3 peering, 7 active+undersized, 21 active+undersized+degraded, 66 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.5 MiB/s wr, 295 op/s; 161/1656 objects degraded (9.722%) 2024-04-26T15:39:10.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:10 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 188/1668 objects degraded (11.271%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:10.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:10 smithi164 ceph-mon[49952]: osdmap e129: 8 total, 8 up, 8 in 2024-04-26T15:39:10.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 258521 9.98 MB/sec execute 514 sec latency 2357.611 ms 2024-04-26T15:39:11.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 258752 9.96 MB/sec execute 515 sec latency 120.455 ms 2024-04-26T15:39:11.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:11 smithi112 ceph-mon[63158]: osdmap e130: 8 total, 8 up, 8 in 2024-04-26T15:39:12.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:11 smithi164 ceph-mon[49952]: osdmap e130: 8 total, 8 up, 8 in 2024-04-26T15:39:12.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 259015 9.95 MB/sec execute 516 sec latency 996.843 ms 2024-04-26T15:39:12.965 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:12 smithi112 ceph-mon[63158]: pgmap v241: 97 pgs: 2 remapped+peering, 3 peering, 6 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 73 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.4 MiB/s wr, 496 op/s; 108/1617 objects degraded (6.679%); 23 KiB/s, 6 objects/s recovering 2024-04-26T15:39:12.965 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:12 smithi112 ceph-mon[63158]: osdmap e131: 8 total, 8 up, 8 in 2024-04-26T15:39:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:12 smithi164 ceph-mon[49952]: pgmap v241: 97 pgs: 2 remapped+peering, 3 peering, 6 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 73 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.4 MiB/s wr, 496 op/s; 108/1617 objects degraded (6.679%); 23 KiB/s, 6 objects/s recovering 2024-04-26T15:39:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:12 smithi164 ceph-mon[49952]: osdmap e131: 8 total, 8 up, 8 in 2024-04-26T15:39:13.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 259273 9.94 MB/sec execute 517 sec latency 50.099 ms 2024-04-26T15:39:13.979 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:13 smithi112 ceph-mon[63158]: osdmap e132: 8 total, 8 up, 8 in 2024-04-26T15:39:14.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:13 smithi164 ceph-mon[49952]: osdmap e132: 8 total, 8 up, 8 in 2024-04-26T15:39:14.776 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:14 smithi112 ceph-mon[63158]: pgmap v244: 97 pgs: 2 remapped+peering, 2 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.5 MiB/s wr, 359 op/s; 29/1608 objects degraded (1.803%); 1.5 MiB/s, 42 objects/s recovering 2024-04-26T15:39:14.777 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:14 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 29/1608 objects degraded (1.803%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:14.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 259525 9.95 MB/sec execute 518 sec latency 49.310 ms 2024-04-26T15:39:15.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:14 smithi164 ceph-mon[49952]: pgmap v244: 97 pgs: 2 remapped+peering, 2 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.5 MiB/s wr, 359 op/s; 29/1608 objects degraded (1.803%); 1.5 MiB/s, 42 objects/s recovering 2024-04-26T15:39:15.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:14 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 29/1608 objects degraded (1.803%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:15.807 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:15 smithi112 ceph-mon[63158]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1608 objects degraded (1.803%), 2 pgs degraded) 2024-04-26T15:39:15.808 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:15 smithi112 ceph-mon[63158]: Cluster is now healthy 2024-04-26T15:39:15.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 259904 9.95 MB/sec execute 519 sec latency 42.148 ms 2024-04-26T15:39:16.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:15 smithi164 ceph-mon[49952]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1608 objects degraded (1.803%), 2 pgs degraded) 2024-04-26T15:39:16.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:15 smithi164 ceph-mon[49952]: Cluster is now healthy 2024-04-26T15:39:16.813 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:16 smithi112 ceph-mon[63158]: pgmap v245: 97 pgs: 1 remapped+peering, 2 peering, 1 active+undersized+remapped, 93 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 3.5 MiB/s wr, 335 op/s; 2.2 MiB/s, 41 objects/s recovering 2024-04-26T15:39:16.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 260239 9.94 MB/sec execute 520 sec latency 66.340 ms 2024-04-26T15:39:17.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:16 smithi164 ceph-mon[49952]: pgmap v245: 97 pgs: 1 remapped+peering, 2 peering, 1 active+undersized+remapped, 93 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 3.5 MiB/s wr, 335 op/s; 2.2 MiB/s, 41 objects/s recovering 2024-04-26T15:39:17.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 260644 9.94 MB/sec execute 521 sec latency 59.167 ms 2024-04-26T15:39:18.862 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:18 smithi112 ceph-mon[63158]: pgmap v246: 97 pgs: 97 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.0 MiB/s wr, 314 op/s; 1.9 MiB/s, 34 objects/s recovering 2024-04-26T15:39:18.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 260851 9.93 MB/sec execute 522 sec latency 43.953 ms 2024-04-26T15:39:19.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:18 smithi164 ceph-mon[49952]: pgmap v246: 97 pgs: 97 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.0 MiB/s wr, 314 op/s; 1.9 MiB/s, 34 objects/s recovering 2024-04-26T15:39:19.873 INFO:teuthology.orchestra.run.smithi112.stdout: 5 261187 9.94 MB/sec execute 523 sec latency 63.848 ms 2024-04-26T15:39:20.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 261506 9.94 MB/sec execute 524 sec latency 72.208 ms 2024-04-26T15:39:20.909 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:20 smithi112 ceph-mon[63158]: pgmap v247: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.0 MiB/s wr, 285 op/s; 1.5 MiB/s, 27 objects/s recovering 2024-04-26T15:39:20.909 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T15:39:21.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:20 smithi164 ceph-mon[49952]: pgmap v247: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.0 MiB/s wr, 285 op/s; 1.5 MiB/s, 27 objects/s recovering 2024-04-26T15:39:21.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:20 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T15:39:21.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:21 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T15:39:21.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:21 smithi164 ceph-mon[49952]: Upgrade: osd.6 is safe to restart 2024-04-26T15:39:21.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 261831 9.94 MB/sec execute 525 sec latency 97.541 ms 2024-04-26T15:39:21.938 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:21 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T15:39:21.938 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:21 smithi112 ceph-mon[63158]: Upgrade: osd.6 is safe to restart 2024-04-26T15:39:22.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 262173 9.93 MB/sec execute 526 sec latency 67.182 ms 2024-04-26T15:39:22.952 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:22 smithi112 ceph-mon[63158]: pgmap v248: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.3 MiB/s wr, 351 op/s; 764 KiB/s, 12 objects/s recovering 2024-04-26T15:39:23.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:22 smithi164 ceph-mon[49952]: pgmap v248: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.3 MiB/s wr, 351 op/s; 764 KiB/s, 12 objects/s recovering 2024-04-26T15:39:23.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 262576 9.93 MB/sec execute 527 sec latency 40.754 ms 2024-04-26T15:39:24.480 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:24 smithi112 ceph-mon[63158]: pgmap v249: 97 pgs: 97 active+clean; 350 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.4 MiB/s wr, 437 op/s; 691 KiB/s, 11 objects/s recovering 2024-04-26T15:39:24.481 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:24 smithi112 ceph-mon[63158]: Upgrade: Updating osd.6 2024-04-26T15:39:24.481 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:24.481 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T15:39:24.481 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:39:24.481 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:24 smithi112 ceph-mon[63158]: Deploying daemon osd.6 on smithi164 2024-04-26T15:39:24.481 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:24.481 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:39:24.481 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:24.508 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:24 smithi164 ceph-mon[49952]: pgmap v249: 97 pgs: 97 active+clean; 350 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.4 MiB/s wr, 437 op/s; 691 KiB/s, 11 objects/s recovering 2024-04-26T15:39:24.508 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:24 smithi164 ceph-mon[49952]: Upgrade: Updating osd.6 2024-04-26T15:39:24.508 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:24.508 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T15:39:24.508 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:39:24.508 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:24 smithi164 ceph-mon[49952]: Deploying daemon osd.6 on smithi164 2024-04-26T15:39:24.508 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:24.509 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:39:24.509 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:24.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 262910 9.93 MB/sec execute 528 sec latency 52.908 ms 2024-04-26T15:39:25.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 263195 9.92 MB/sec execute 529 sec latency 56.820 ms 2024-04-26T15:39:26.532 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:26 smithi112 ceph-mon[63158]: pgmap v250: 97 pgs: 97 active+clean; 355 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 480 op/s; 552 KiB/s, 6 objects/s recovering 2024-04-26T15:39:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:26 smithi164 ceph-mon[49952]: pgmap v250: 97 pgs: 97 active+clean; 355 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 480 op/s; 552 KiB/s, 6 objects/s recovering 2024-04-26T15:39:26.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 263554 9.93 MB/sec execute 530 sec latency 64.840 ms 2024-04-26T15:39:27.313 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:27 smithi164 systemd[1]: Stopping Ceph osd.6 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:39:27.560 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:27 smithi112 ceph-mon[63158]: osd.6 marked itself down and dead 2024-04-26T15:39:27.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:27 smithi164 ceph-mon[49952]: osd.6 marked itself down and dead 2024-04-26T15:39:27.597 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:27 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[34280]: 2024-04-26T15:39:27.313+0000 7fa647e3f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:39:27.598 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:27 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[34280]: 2024-04-26T15:39:27.313+0000 7fa647e3f700 -1 osd.6 132 *** Got signal Terminated *** 2024-04-26T15:39:27.598 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:27 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[34280]: 2024-04-26T15:39:27.313+0000 7fa647e3f700 -1 osd.6 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:39:27.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 263885 9.92 MB/sec execute 531 sec latency 479.876 ms 2024-04-26T15:39:28.296 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:28 smithi164 podman[59016]: 2024-04-26 15:39:28.02422278 +0000 UTC m=+0.835423584 container died d77044ffb3d6f26be917e2c47a9f177adb112090816b78894e92b1d1b546c65e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., RELEASE=HEAD, architecture=x86_64, distribution-scope=public, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_CLEAN=True) 2024-04-26T15:39:28.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:28 smithi164 ceph-mon[49952]: pgmap v251: 97 pgs: 97 active+clean; 351 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.8 MiB/s wr, 515 op/s; 60 KiB/s, 1 objects/s recovering 2024-04-26T15:39:28.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:28 smithi164 ceph-mon[49952]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:39:28.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:28 smithi164 ceph-mon[49952]: osdmap e133: 8 total, 7 up, 8 in 2024-04-26T15:39:28.598 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:28 smithi164 podman[59016]: 2024-04-26 15:39:28.488114287 +0000 UTC m=+1.299315096 container cleanup d77044ffb3d6f26be917e2c47a9f177adb112090816b78894e92b1d1b546c65e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, RELEASE=HEAD, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-26T15:39:28.598 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:28 smithi164 bash[59016]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6 2024-04-26T15:39:28.619 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:28 smithi112 ceph-mon[63158]: pgmap v251: 97 pgs: 97 active+clean; 351 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.8 MiB/s wr, 515 op/s; 60 KiB/s, 1 objects/s recovering 2024-04-26T15:39:28.619 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:28 smithi112 ceph-mon[63158]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:39:28.619 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:28 smithi112 ceph-mon[63158]: osdmap e133: 8 total, 7 up, 8 in 2024-04-26T15:39:28.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 264033 9.91 MB/sec execute 532 sec latency 1008.677 ms 2024-04-26T15:39:29.097 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:28 smithi164 podman[59032]: 2024-04-26 15:39:28.8065738 +0000 UTC m=+0.782954988 container remove d77044ffb3d6f26be917e2c47a9f177adb112090816b78894e92b1d1b546c65e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, distribution-scope=public, io.openshift.expose-services=, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_CLEAN=True, name=centos-stream, vcs-type=git, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, architecture=x86_64, version=8, io.openshift.tags=base centos centos-stream) 2024-04-26T15:39:29.518 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:29 smithi164 ceph-mon[49952]: osdmap e134: 8 total, 7 up, 8 in 2024-04-26T15:39:29.518 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:29 smithi164 podman[59121]: 2024-04-26 15:39:29.095076519 +0000 UTC m=+0.020517687 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:39:29.518 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:29 smithi164 podman[59121]: 2024-04-26 15:39:29.224623084 +0000 UTC m=+0.150064233 container create ce8ee276d7ef7977fa4c86dc3babba27f6d148a2e9b10bc84dcfa6ae0f14d557 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True) 2024-04-26T15:39:29.632 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:29 smithi112 ceph-mon[63158]: osdmap e134: 8 total, 7 up, 8 in 2024-04-26T15:39:29.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:29 smithi164 podman[59121]: 2024-04-26 15:39:29.519166843 +0000 UTC m=+0.444608010 container init ce8ee276d7ef7977fa4c86dc3babba27f6d148a2e9b10bc84dcfa6ae0f14d557 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:39:29.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:29 smithi164 podman[59121]: 2024-04-26 15:39:29.523694437 +0000 UTC m=+0.449135590 container start ce8ee276d7ef7977fa4c86dc3babba27f6d148a2e9b10bc84dcfa6ae0f14d557 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True) 2024-04-26T15:39:29.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:29 smithi164 podman[59121]: 2024-04-26 15:39:29.611007542 +0000 UTC m=+0.536448694 container attach ce8ee276d7ef7977fa4c86dc3babba27f6d148a2e9b10bc84dcfa6ae0f14d557 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:39:29.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 264382 9.91 MB/sec execute 533 sec latency 37.929 ms 2024-04-26T15:39:30.226 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:29 smithi164 podman[59121]: 2024-04-26 15:39:29.873085086 +0000 UTC m=+0.798526244 container died ce8ee276d7ef7977fa4c86dc3babba27f6d148a2e9b10bc84dcfa6ae0f14d557 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T15:39:30.406 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:30 smithi112 ceph-mon[63158]: pgmap v254: 97 pgs: 12 peering, 5 stale+active+clean, 80 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.9 MiB/s wr, 467 op/s 2024-04-26T15:39:30.406 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:30 smithi112 ceph-mon[63158]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-26T15:39:30.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:30 smithi164 ceph-mon[49952]: pgmap v254: 97 pgs: 12 peering, 5 stale+active+clean, 80 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.9 MiB/s wr, 467 op/s 2024-04-26T15:39:30.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:30 smithi164 ceph-mon[49952]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-26T15:39:30.852 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:30 smithi164 podman[59146]: 2024-04-26 15:39:30.599855989 +0000 UTC m=+0.721315324 container remove ce8ee276d7ef7977fa4c86dc3babba27f6d148a2e9b10bc84dcfa6ae0f14d557 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-26T15:39:30.852 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:30 smithi164 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-26T15:39:30.852 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:30 smithi164 systemd[1]: Stopped Ceph osd.6 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:39:30.852 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:30 smithi164 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.6.service: Consumed 4min 50.192s CPU time. 2024-04-26T15:39:30.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 264799 9.91 MB/sec execute 534 sec latency 34.983 ms 2024-04-26T15:39:31.109 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:30 smithi164 systemd[1]: Starting Ceph osd.6 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:39:31.109 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 podman[59252]: 2024-04-26 15:39:30.986776857 +0000 UTC m=+0.019337063 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:39:31.437 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 podman[59252]: 2024-04-26 15:39:31.109256142 +0000 UTC m=+0.141816338 container create 1979cfd0f765a32924999a596c7ed0651672bb36fa28bf3b0301b8a2e4ad176b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD) 2024-04-26T15:39:31.438 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 podman[59252]: 2024-04-26 15:39:31.345469983 +0000 UTC m=+0.378030194 container init 1979cfd0f765a32924999a596c7ed0651672bb36fa28bf3b0301b8a2e4ad176b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-26T15:39:31.438 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 podman[59252]: 2024-04-26 15:39:31.349919529 +0000 UTC m=+0.382479729 container start 1979cfd0f765a32924999a596c7ed0651672bb36fa28bf3b0301b8a2e4ad176b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-26T15:39:31.689 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:31 smithi112 ceph-mon[63158]: Health check failed: Degraded data redundancy: 76/1602 objects degraded (4.744%), 10 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:31.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:31 smithi164 ceph-mon[49952]: Health check failed: Degraded data redundancy: 76/1602 objects degraded (4.744%), 10 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:31.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 podman[59252]: 2024-04-26 15:39:31.445641412 +0000 UTC m=+0.478201625 container attach 1979cfd0f765a32924999a596c7ed0651672bb36fa28bf3b0301b8a2e4ad176b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True) 2024-04-26T15:39:31.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 265166 9.91 MB/sec execute 535 sec latency 57.074 ms 2024-04-26T15:39:32.324 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate[59266]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:39:32.324 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate[59266]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:39:32.324 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 bash[59252]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:39:32.324 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 bash[59252]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 bash[59252]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 bash[59252]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 bash[59252]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 bash[59252]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 bash[59252]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 bash[59252]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 bash[59252]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate[59266]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate[59266]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate[59266]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate[59266]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate[59266]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-26T15:39:32.325 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate[59266]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-26T15:39:32.326 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate[59266]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-26T15:39:32.326 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:31 smithi164 podman[59252]: 2024-04-26 15:39:31.946799367 +0000 UTC m=+0.979359564 container died 1979cfd0f765a32924999a596c7ed0651672bb36fa28bf3b0301b8a2e4ad176b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD) 2024-04-26T15:39:32.454 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:32 smithi112 ceph-mon[63158]: pgmap v255: 97 pgs: 2 active+undersized, 10 active+undersized+degraded, 12 peering, 3 stale+active+clean, 70 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.2 MiB/s wr, 422 op/s; 76/1602 objects degraded (4.744%) 2024-04-26T15:39:32.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:32 smithi164 ceph-mon[49952]: pgmap v255: 97 pgs: 2 active+undersized, 10 active+undersized+degraded, 12 peering, 3 stale+active+clean, 70 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.2 MiB/s wr, 422 op/s; 76/1602 objects degraded (4.744%) 2024-04-26T15:39:32.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 265471 9.92 MB/sec execute 536 sec latency 55.005 ms 2024-04-26T15:39:33.097 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:32 smithi164 podman[59404]: 2024-04-26 15:39:32.751395157 +0000 UTC m=+0.798858788 container remove 1979cfd0f765a32924999a596c7ed0651672bb36fa28bf3b0301b8a2e4ad176b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:39:33.097 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:32 smithi164 podman[59462]: 2024-04-26 15:39:32.870711095 +0000 UTC m=+0.020125852 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:39:33.098 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:32 smithi164 podman[59462]: 2024-04-26 15:39:32.994045753 +0000 UTC m=+0.143460475 container create 78ef3c811a10785f20696f954c9884235e247393a97d1bda3041699372f071da (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:39:33.725 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:33 smithi112 ceph-mon[63158]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-26T15:39:33.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:33 smithi164 ceph-mon[49952]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-26T15:39:33.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:33 smithi164 podman[59462]: 2024-04-26 15:39:33.605128978 +0000 UTC m=+0.754543713 container init 78ef3c811a10785f20696f954c9884235e247393a97d1bda3041699372f071da (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux ) 2024-04-26T15:39:33.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:33 smithi164 podman[59462]: 2024-04-26 15:39:33.608340987 +0000 UTC m=+0.757755712 container start 78ef3c811a10785f20696f954c9884235e247393a97d1bda3041699372f071da (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-26T15:39:33.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:33 smithi164 ceph-osd[59481]: -- 172.21.15.164:0/3504628140 <== mon.1 v2:172.21.15.164:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563ec3e61680 con 0x563ec3e2b000 2024-04-26T15:39:33.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:33 smithi164 bash[59462]: 78ef3c811a10785f20696f954c9884235e247393a97d1bda3041699372f071da 2024-04-26T15:39:33.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:33 smithi164 systemd[1]: Started Ceph osd.6 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:39:33.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 265740 9.91 MB/sec execute 537 sec latency 55.312 ms 2024-04-26T15:39:34.597 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:34 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:39:34.152+0000 7f93544727c0 -1 Falling back to public interface 2024-04-26T15:39:34.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 265996 9.90 MB/sec execute 538 sec latency 54.647 ms 2024-04-26T15:39:34.989 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:34 smithi112 ceph-mon[63158]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.1 MiB/s wr, 404 op/s; 191/1644 objects degraded (11.618%) 2024-04-26T15:39:34.989 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:34.990 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:34.990 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:39:35.069 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:34 smithi164 ceph-mon[49952]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.1 MiB/s wr, 404 op/s; 191/1644 objects degraded (11.618%) 2024-04-26T15:39:35.069 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:35.069 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:35.069 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:39:35.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 266280 9.89 MB/sec execute 539 sec latency 64.300 ms 2024-04-26T15:39:36.774 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:36 smithi112 ceph-mon[63158]: pgmap v257: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.7 MiB/s wr, 417 op/s; 198/1680 objects degraded (11.786%) 2024-04-26T15:39:36.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 266517 9.89 MB/sec execute 540 sec latency 49.172 ms 2024-04-26T15:39:36.900 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:39:37.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:36 smithi164 ceph-mon[49952]: pgmap v257: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.7 MiB/s wr, 417 op/s; 198/1680 objects degraded (11.786%) 2024-04-26T15:39:37.796 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:37 smithi112 ceph-mon[63158]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:37.806 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:39:37.806 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (8m) 113s ago 17m 25.1M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:39:37.806 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (5m) 113s ago 17m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:39:37.806 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (5m) 38s ago 16m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:39:37.806 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (8m) 113s ago 17m 54.1M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:39:37.806 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (11m) 113s ago 11m 38.3M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:39:37.806 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (11m) 38s ago 11m 14.8M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:39:37.806 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (7m) 113s ago 18m 522M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:39:37.806 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (6m) 38s ago 16m 468M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:39:37.806 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (6m) 113s ago 18m 52.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:39:37.806 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (6m) 38s ago 16m 45.5M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:39:37.806 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (11m) 113s ago 11m 266M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:39:37.807 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (11m) 38s ago 11m 90.2M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:39:37.807 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (8m) 113s ago 17m 17.5M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:39:37.807 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (8m) 38s ago 16m 18.1M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:39:37.807 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (4m) 113s ago 16m 353M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:39:37.807 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (3m) 113s ago 15m 291M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:39:37.807 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (2m) 113s ago 15m 213M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:39:37.807 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (2m) 113s ago 14m 14.9M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd 27fefb9c15b3 2024-04-26T15:39:37.807 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (80s) 38s ago 14m 266M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd b68352a89f2e 2024-04-26T15:39:37.807 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (42s) 38s ago 13m 14.7M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 61be8ee4a0ab 2024-04-26T15:39:37.807 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 starting - - - 2704M 2024-04-26T15:39:37.807 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (13m) 38s ago 13m 804M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:39:37.807 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (7m) 113s ago 16m 73.2M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:39:37.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 266789 9.89 MB/sec execute 541 sec latency 81.911 ms 2024-04-26T15:39:38.038 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:37 smithi164 ceph-mon[49952]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:38.358 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:39:38.358 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:39:38.358 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:39:38.358 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 6 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 10 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:39:38.359 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:39:38.820 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:39:38.820 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:39:38.820 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:39:38.820 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:39:38.820 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:39:38.820 INFO:teuthology.orchestra.run.smithi112.stdout: "crash", 2024-04-26T15:39:38.821 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:39:38.821 INFO:teuthology.orchestra.run.smithi112.stdout: "mon" 2024-04-26T15:39:38.821 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:39:38.821 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "13/23 daemons upgraded", 2024-04-26T15:39:38.821 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T15:39:38.821 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:39:38.821 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:39:38.826 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:38 smithi112 ceph-mon[63158]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 467 op/s; 193/1665 objects degraded (11.592%) 2024-04-26T15:39:38.826 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:38 smithi112 ceph-mon[63158]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:38.826 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:38 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 193/1665 objects degraded (11.592%), 25 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:38.826 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:38 smithi112 ceph-mon[63158]: from='client.44307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:38.827 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:38 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:38.827 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:38 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:38.827 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:38 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/1880202305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:38.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 267138 9.89 MB/sec execute 542 sec latency 46.762 ms 2024-04-26T15:39:39.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:38 smithi164 ceph-mon[49952]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 467 op/s; 193/1665 objects degraded (11.592%) 2024-04-26T15:39:39.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:38 smithi164 ceph-mon[49952]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:39.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:38 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 193/1665 objects degraded (11.592%), 25 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:39.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:38 smithi164 ceph-mon[49952]: from='client.44307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:39.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:38 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:39.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:38 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:39.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:38 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/1880202305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:39.390 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 193/1665 objects degraded (11.592%), 25 pgs degraded 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout: osd.6 (root=default,host=smithi164) is down 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 193/1665 objects degraded (11.592%), 25 pgs degraded 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-04-26T15:39:39.391 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-04-26T15:39:39.392 INFO:teuthology.orchestra.run.smithi112.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-04-26T15:39:39.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 267450 9.88 MB/sec execute 543 sec latency 63.589 ms 2024-04-26T15:39:39.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:39 smithi112 ceph-mon[63158]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:39.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:39.888 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:39:39.888 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:39 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/2600035458' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:39:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:39 smithi164 ceph-mon[49952]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:39:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:39:40.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:39 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/2600035458' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:39:40.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:40 smithi164 ceph-mon[49952]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 351 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.2 MiB/s wr, 416 op/s; 188/1617 objects degraded (11.626%) 2024-04-26T15:39:40.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:40.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:40.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 267828 9.88 MB/sec execute 544 sec latency 53.253 ms 2024-04-26T15:39:40.910 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:40 smithi112 ceph-mon[63158]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 351 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.2 MiB/s wr, 416 op/s; 188/1617 objects degraded (11.626%) 2024-04-26T15:39:40.910 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:40.911 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:41.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 268106 9.87 MB/sec execute 545 sec latency 52.554 ms 2024-04-26T15:39:42.025 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:41 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:39:41.706+0000 7f93544727c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-26T15:39:42.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 268471 9.88 MB/sec execute 546 sec latency 56.085 ms 2024-04-26T15:39:42.943 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:42 smithi112 ceph-mon[63158]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.8 MiB/s wr, 507 op/s; 192/1623 objects degraded (11.830%) 2024-04-26T15:39:43.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:42 smithi164 ceph-mon[49952]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.8 MiB/s wr, 507 op/s; 192/1623 objects degraded (11.830%) 2024-04-26T15:39:43.425 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:43 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:39:43.101+0000 7f93544727c0 -1 osd.6 132 log_to_monitors true 2024-04-26T15:39:43.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 268843 9.88 MB/sec execute 547 sec latency 34.864 ms 2024-04-26T15:39:43.952 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:43 smithi112 ceph-mon[63158]: from='osd.6 [v2:172.21.15.164:6816/1212040165,v1:172.21.15.164:6817/1212040165]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:39:43.952 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:43 smithi112 ceph-mon[63158]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:39:43.952 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:43 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:43.953 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:43 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:43.953 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:43 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:39:43.953 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:43 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:39:44.097 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:39:43 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:39:43.879+0000 7f93458ca700 -1 osd.6 132 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:39:44.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:43 smithi164 ceph-mon[49952]: from='osd.6 [v2:172.21.15.164:6816/1212040165,v1:172.21.15.164:6817/1212040165]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:39:44.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:43 smithi164 ceph-mon[49952]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:39:44.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:43 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:44.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:43 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:44.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:43 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:39:44.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:43 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:39:44.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 269143 9.88 MB/sec execute 548 sec latency 51.475 ms 2024-04-26T15:39:45.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.9 MiB/s wr, 478 op/s; 195/1668 objects degraded (11.691%) 2024-04-26T15:39:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 195/1668 objects degraded (11.691%), 25 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T15:39:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: osdmap e135: 8 total, 7 up, 8 in 2024-04-26T15:39:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: from='osd.6 [v2:172.21.15.164:6816/1212040165,v1:172.21.15.164:6817/1212040165]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:39:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:39:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:39:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:45.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T15:39:45.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T15:39:45.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:44 smithi164 ceph-mon[49952]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-26T15:39:45.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.9 MiB/s wr, 478 op/s; 195/1668 objects degraded (11.691%) 2024-04-26T15:39:45.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:45.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 195/1668 objects degraded (11.691%), 25 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:45.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T15:39:45.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: osdmap e135: 8 total, 7 up, 8 in 2024-04-26T15:39:45.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: from='osd.6 [v2:172.21.15.164:6816/1212040165,v1:172.21.15.164:6817/1212040165]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:39:45.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:45.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:39:45.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:39:45.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:45.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:45.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:39:45.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T15:39:45.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T15:39:45.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:44 smithi112 ceph-mon[63158]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-26T15:39:45.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 269146 9.87 MB/sec execute 549 sec latency 1013.284 ms 2024-04-26T15:39:46.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:45 smithi164 ceph-mon[49952]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:39:46.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:45 smithi164 ceph-mon[49952]: osd.6 [v2:172.21.15.164:6816/1212040165,v1:172.21.15.164:6817/1212040165] boot 2024-04-26T15:39:46.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:45 smithi164 ceph-mon[49952]: osdmap e136: 8 total, 8 up, 8 in 2024-04-26T15:39:46.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:39:46.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:45 smithi112 ceph-mon[63158]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:39:46.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:45 smithi112 ceph-mon[63158]: osd.6 [v2:172.21.15.164:6816/1212040165,v1:172.21.15.164:6817/1212040165] boot 2024-04-26T15:39:46.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:45 smithi112 ceph-mon[63158]: osdmap e136: 8 total, 8 up, 8 in 2024-04-26T15:39:46.212 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:39:46.876 INFO:teuthology.orchestra.run.smithi112.stdout: 5 269146 9.85 MB/sec execute 550 sec latency 2013.324 ms 2024-04-26T15:39:47.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:46 smithi112 ceph-mon[63158]: pgmap v264: 97 pgs: 4 peering, 10 active+undersized, 21 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.5 MiB/s wr, 380 op/s; 159/1629 objects degraded (9.761%) 2024-04-26T15:39:47.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:46 smithi112 ceph-mon[63158]: osdmap e137: 8 total, 8 up, 8 in 2024-04-26T15:39:47.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:46 smithi164 ceph-mon[49952]: pgmap v264: 97 pgs: 4 peering, 10 active+undersized, 21 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.5 MiB/s wr, 380 op/s; 159/1629 objects degraded (9.761%) 2024-04-26T15:39:47.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:46 smithi164 ceph-mon[49952]: osdmap e137: 8 total, 8 up, 8 in 2024-04-26T15:39:47.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 269386 9.84 MB/sec execute 551 sec latency 2286.265 ms 2024-04-26T15:39:47.878 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:47 smithi112 ceph-mon[63158]: osdmap e138: 8 total, 8 up, 8 in 2024-04-26T15:39:48.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:47 smithi164 ceph-mon[49952]: osdmap e138: 8 total, 8 up, 8 in 2024-04-26T15:39:48.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 269395 9.82 MB/sec execute 552 sec latency 999.990 ms 2024-04-26T15:39:49.131 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:48 smithi112 ceph-mon[63158]: pgmap v267: 97 pgs: 2 remapped+peering, 4 peering, 8 active+undersized, 14 active+undersized+degraded, 69 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 4.2 MiB/s wr, 442 op/s; 100/1641 objects degraded (6.094%); 199 KiB/s, 71 keys/s, 8 objects/s recovering 2024-04-26T15:39:49.132 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:48 smithi112 ceph-mon[63158]: osdmap e139: 8 total, 8 up, 8 in 2024-04-26T15:39:49.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:48 smithi164 ceph-mon[49952]: pgmap v267: 97 pgs: 2 remapped+peering, 4 peering, 8 active+undersized, 14 active+undersized+degraded, 69 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 4.2 MiB/s wr, 442 op/s; 100/1641 objects degraded (6.094%); 199 KiB/s, 71 keys/s, 8 objects/s recovering 2024-04-26T15:39:49.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:48 smithi164 ceph-mon[49952]: osdmap e139: 8 total, 8 up, 8 in 2024-04-26T15:39:49.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 269703 9.81 MB/sec execute 553 sec latency 1040.880 ms 2024-04-26T15:39:50.140 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:49 smithi112 ceph-mon[63158]: osdmap e140: 8 total, 8 up, 8 in 2024-04-26T15:39:50.140 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:49 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 100/1641 objects degraded (6.094%), 14 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:50.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:49 smithi164 ceph-mon[49952]: osdmap e140: 8 total, 8 up, 8 in 2024-04-26T15:39:50.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:49 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 100/1641 objects degraded (6.094%), 14 pgs degraded (PG_DEGRADED) 2024-04-26T15:39:50.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 270011 9.81 MB/sec execute 554 sec latency 40.635 ms 2024-04-26T15:39:50.924 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:50 smithi112 ceph-mon[63158]: pgmap v270: 97 pgs: 1 remapped+peering, 5 peering, 2 active+undersized, 1 active+undersized+remapped, 4 active+undersized+degraded, 84 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.9 MiB/s wr, 323 op/s; 38/1623 objects degraded (2.341%); 796 KiB/s, 71 keys/s, 24 objects/s recovering 2024-04-26T15:39:51.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:50 smithi164 ceph-mon[49952]: pgmap v270: 97 pgs: 1 remapped+peering, 5 peering, 2 active+undersized, 1 active+undersized+remapped, 4 active+undersized+degraded, 84 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.9 MiB/s wr, 323 op/s; 38/1623 objects degraded (2.341%); 796 KiB/s, 71 keys/s, 24 objects/s recovering 2024-04-26T15:39:51.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 270343 9.81 MB/sec execute 555 sec latency 41.843 ms 2024-04-26T15:39:51.953 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:51 smithi112 ceph-mon[63158]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/1623 objects degraded (2.341%), 4 pgs degraded) 2024-04-26T15:39:51.953 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:51 smithi112 ceph-mon[63158]: Cluster is now healthy 2024-04-26T15:39:52.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:51 smithi164 ceph-mon[49952]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/1623 objects degraded (2.341%), 4 pgs degraded) 2024-04-26T15:39:52.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:51 smithi164 ceph-mon[49952]: Cluster is now healthy 2024-04-26T15:39:52.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 270767 9.81 MB/sec execute 556 sec latency 48.105 ms 2024-04-26T15:39:52.974 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:52 smithi112 ceph-mon[63158]: pgmap v271: 97 pgs: 1 remapped+peering, 1 peering, 1 active+undersized+remapped, 94 active+clean; 351 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.3 MiB/s wr, 196 op/s; 1.9 MiB/s, 54 keys/s, 34 objects/s recovering 2024-04-26T15:39:53.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:52 smithi164 ceph-mon[49952]: pgmap v271: 97 pgs: 1 remapped+peering, 1 peering, 1 active+undersized+remapped, 94 active+clean; 351 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.3 MiB/s wr, 196 op/s; 1.9 MiB/s, 54 keys/s, 34 objects/s recovering 2024-04-26T15:39:53.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 271201 9.81 MB/sec execute 557 sec latency 37.629 ms 2024-04-26T15:39:54.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 271504 9.81 MB/sec execute 558 sec latency 66.947 ms 2024-04-26T15:39:55.025 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:54 smithi112 ceph-mon[63158]: pgmap v272: 97 pgs: 1 peering, 96 active+clean; 349 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.2 MiB/s wr, 183 op/s; 1.6 MiB/s, 26 objects/s recovering 2024-04-26T15:39:55.026 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:55.026 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:39:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:54 smithi164 ceph-mon[49952]: pgmap v272: 97 pgs: 1 peering, 96 active+clean; 349 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.2 MiB/s wr, 183 op/s; 1.6 MiB/s, 26 objects/s recovering 2024-04-26T15:39:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:39:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:39:55.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 271880 9.81 MB/sec execute 559 sec latency 50.783 ms 2024-04-26T15:39:56.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 272239 9.81 MB/sec execute 560 sec latency 37.201 ms 2024-04-26T15:39:57.081 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:56 smithi112 ceph-mon[63158]: pgmap v273: 97 pgs: 97 active+clean; 353 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 3.4 MiB/s wr, 338 op/s; 1.3 MiB/s, 22 objects/s recovering 2024-04-26T15:39:57.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:56 smithi164 ceph-mon[49952]: pgmap v273: 97 pgs: 97 active+clean; 353 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 3.4 MiB/s wr, 338 op/s; 1.3 MiB/s, 22 objects/s recovering 2024-04-26T15:39:57.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 272556 9.81 MB/sec execute 561 sec latency 44.133 ms 2024-04-26T15:39:58.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 272848 9.81 MB/sec execute 562 sec latency 75.519 ms 2024-04-26T15:39:59.143 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:58 smithi112 ceph-mon[63158]: pgmap v274: 97 pgs: 97 active+clean; 349 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.2 MiB/s wr, 312 op/s; 967 KiB/s, 15 objects/s recovering 2024-04-26T15:39:59.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:58 smithi164 ceph-mon[49952]: pgmap v274: 97 pgs: 97 active+clean; 349 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.2 MiB/s wr, 312 op/s; 967 KiB/s, 15 objects/s recovering 2024-04-26T15:39:59.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 273186 9.81 MB/sec execute 563 sec latency 41.355 ms 2024-04-26T15:40:00.173 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:59 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T15:40:00.173 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:59 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T15:40:00.174 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:39:59 smithi112 ceph-mon[63158]: Upgrade: osd.7 is safe to restart 2024-04-26T15:40:00.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:59 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T15:40:00.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:59 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T15:40:00.280 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:39:59 smithi164 ceph-mon[49952]: Upgrade: osd.7 is safe to restart 2024-04-26T15:40:00.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 273490 9.80 MB/sec execute 564 sec latency 44.955 ms 2024-04-26T15:40:01.179 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:00 smithi112 ceph-mon[63158]: pgmap v275: 97 pgs: 97 active+clean; 353 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.7 MiB/s wr, 417 op/s; 828 KiB/s, 11 objects/s recovering 2024-04-26T15:40:01.179 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:00 smithi112 ceph-mon[63158]: overall HEALTH_OK 2024-04-26T15:40:01.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:00 smithi164 ceph-mon[49952]: pgmap v275: 97 pgs: 97 active+clean; 353 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.7 MiB/s wr, 417 op/s; 828 KiB/s, 11 objects/s recovering 2024-04-26T15:40:01.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:00 smithi164 ceph-mon[49952]: overall HEALTH_OK 2024-04-26T15:40:01.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 273800 9.80 MB/sec execute 565 sec latency 45.777 ms 2024-04-26T15:40:02.478 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:02 smithi112 ceph-mon[63158]: pgmap v276: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 468 op/s; 704 KiB/s, 9 objects/s recovering 2024-04-26T15:40:02.479 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:02 smithi112 ceph-mon[63158]: Upgrade: Updating osd.7 2024-04-26T15:40:02.479 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:02 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:02.479 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:02 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T15:40:02.479 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:02 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:40:02.479 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:02 smithi112 ceph-mon[63158]: Deploying daemon osd.7 on smithi164 2024-04-26T15:40:02.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:02 smithi164 ceph-mon[49952]: pgmap v276: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 468 op/s; 704 KiB/s, 9 objects/s recovering 2024-04-26T15:40:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:02 smithi164 ceph-mon[49952]: Upgrade: Updating osd.7 2024-04-26T15:40:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:02 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:02 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T15:40:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:02 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:40:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:02 smithi164 ceph-mon[49952]: Deploying daemon osd.7 on smithi164 2024-04-26T15:40:02.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 274109 9.79 MB/sec execute 566 sec latency 70.887 ms 2024-04-26T15:40:03.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 274461 9.80 MB/sec execute 567 sec latency 53.897 ms 2024-04-26T15:40:04.512 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:04 smithi112 ceph-mon[63158]: pgmap v277: 97 pgs: 97 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.9 MiB/s wr, 454 op/s; 117 KiB/s, 2 objects/s recovering 2024-04-26T15:40:04.512 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:04 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:04.587 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:04 smithi164 ceph-mon[49952]: pgmap v277: 97 pgs: 97 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.9 MiB/s wr, 454 op/s; 117 KiB/s, 2 objects/s recovering 2024-04-26T15:40:04.587 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:04 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:04.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 274797 9.79 MB/sec execute 568 sec latency 47.314 ms 2024-04-26T15:40:05.252 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:04 smithi164 systemd[1]: Stopping Ceph osd.7 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:40:05.522 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:05 smithi112 ceph-mon[63158]: osd.7 marked itself down and dead 2024-04-26T15:40:05.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:05 smithi164 ceph-mon[49952]: osd.7 marked itself down and dead 2024-04-26T15:40:05.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:05 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[37344]: 2024-04-26T15:40:05.251+0000 7f28e755e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:40:05.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:05 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[37344]: 2024-04-26T15:40:05.251+0000 7f28e755e700 -1 osd.7 140 *** Got signal Terminated *** 2024-04-26T15:40:05.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:05 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[37344]: 2024-04-26T15:40:05.251+0000 7f28e755e700 -1 osd.7 140 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:40:05.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 275046 9.78 MB/sec execute 569 sec latency 350.103 ms 2024-04-26T15:40:06.462 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:06 smithi164 podman[61288]: 2024-04-26 15:40:06.115654133 +0000 UTC m=+0.990394305 container died ab696f7cadf935aba14e8bb9b68b9fd4a1b36a7c1669ef984315bedcccb49bee (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.openshift.expose-services=, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, release=754, io.buildah.version=1.19.8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vcs-type=git, maintainer=Guillaume Abrioux ) 2024-04-26T15:40:06.716 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:06 smithi112 ceph-mon[63158]: pgmap v278: 97 pgs: 97 active+clean; 355 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.5 MiB/s wr, 590 op/s; 59 KiB/s, 1 objects/s recovering 2024-04-26T15:40:07.000 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:06 smithi112 ceph-mon[63158]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:40:07.000 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:06 smithi112 ceph-mon[63158]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-26T15:40:07.001 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:06 smithi112 ceph-mon[63158]: osdmap e141: 8 total, 7 up, 8 in 2024-04-26T15:40:07.001 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:06 smithi164 podman[61288]: 2024-04-26 15:40:06.584978339 +0000 UTC m=+1.459718511 container cleanup ab696f7cadf935aba14e8bb9b68b9fd4a1b36a7c1669ef984315bedcccb49bee (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, ceph=True, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, release=754, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD) 2024-04-26T15:40:07.002 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:06 smithi164 bash[61288]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7 2024-04-26T15:40:07.002 INFO:teuthology.orchestra.run.smithi112.stdout: 5 275202 9.77 MB/sec execute 570 sec latency 973.133 ms 2024-04-26T15:40:07.002 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:06 smithi164 ceph-mon[49952]: pgmap v278: 97 pgs: 97 active+clean; 355 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.5 MiB/s wr, 590 op/s; 59 KiB/s, 1 objects/s recovering 2024-04-26T15:40:07.002 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:06 smithi164 ceph-mon[49952]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T15:40:07.002 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:06 smithi164 ceph-mon[49952]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-26T15:40:07.002 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:06 smithi164 ceph-mon[49952]: osdmap e141: 8 total, 7 up, 8 in 2024-04-26T15:40:07.237 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:06 smithi164 podman[61304]: 2024-04-26 15:40:06.895107295 +0000 UTC m=+0.779228179 container remove ab696f7cadf935aba14e8bb9b68b9fd4a1b36a7c1669ef984315bedcccb49bee (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, release=754, GIT_CLEAN=True, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, version=8) 2024-04-26T15:40:07.502 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:07 smithi164 podman[61396]: 2024-04-26 15:40:07.143023479 +0000 UTC m=+0.020466813 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:40:07.503 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:07 smithi164 podman[61396]: 2024-04-26 15:40:07.26282774 +0000 UTC m=+0.140271059 container create 93e0ea1dafe685d91d8f7fb9366e7a0a0f36e00fce621199e0078b2677eaf472 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-26T15:40:07.742 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:07 smithi112 ceph-mon[63158]: osdmap e142: 8 total, 7 up, 8 in 2024-04-26T15:40:07.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:07 smithi164 ceph-mon[49952]: osdmap e142: 8 total, 7 up, 8 in 2024-04-26T15:40:07.848 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:07 smithi164 podman[61396]: 2024-04-26 15:40:07.532547781 +0000 UTC m=+0.409991100 container init 93e0ea1dafe685d91d8f7fb9366e7a0a0f36e00fce621199e0078b2677eaf472 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240424, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-26T15:40:07.848 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:07 smithi164 podman[61396]: 2024-04-26 15:40:07.536745393 +0000 UTC m=+0.414188726 container start 93e0ea1dafe685d91d8f7fb9366e7a0a0f36e00fce621199e0078b2677eaf472 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T15:40:07.848 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:07 smithi164 podman[61396]: 2024-04-26 15:40:07.632837173 +0000 UTC m=+0.510280515 container attach 93e0ea1dafe685d91d8f7fb9366e7a0a0f36e00fce621199e0078b2677eaf472 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-26T15:40:07.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 275613 9.78 MB/sec execute 571 sec latency 84.340 ms 2024-04-26T15:40:08.253 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:07 smithi164 podman[61396]: 2024-04-26 15:40:07.883380265 +0000 UTC m=+0.760823583 container died 93e0ea1dafe685d91d8f7fb9366e7a0a0f36e00fce621199e0078b2677eaf472 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD) 2024-04-26T15:40:08.750 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:08 smithi112 ceph-mon[63158]: pgmap v281: 97 pgs: 12 peering, 9 stale+active+clean, 76 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.4 MiB/s wr, 507 op/s 2024-04-26T15:40:08.801 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:08 smithi164 ceph-mon[49952]: pgmap v281: 97 pgs: 12 peering, 9 stale+active+clean, 76 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.4 MiB/s wr, 507 op/s 2024-04-26T15:40:08.801 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:08 smithi164 podman[61422]: 2024-04-26 15:40:08.570874175 +0000 UTC m=+0.682080875 container remove 93e0ea1dafe685d91d8f7fb9366e7a0a0f36e00fce621199e0078b2677eaf472 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T15:40:08.801 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:08 smithi164 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.7.service: Deactivated successfully. 2024-04-26T15:40:08.801 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:08 smithi164 systemd[1]: Stopped Ceph osd.7 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:40:08.801 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:08 smithi164 systemd[1]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.7.service: Consumed 6min 56.340s CPU time. 2024-04-26T15:40:08.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 276038 9.79 MB/sec execute 572 sec latency 35.168 ms 2024-04-26T15:40:09.097 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:08 smithi164 systemd[1]: Starting Ceph osd.7 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:40:09.097 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:09 smithi164 podman[61526]: 2024-04-26 15:40:08.931083797 +0000 UTC m=+0.020855520 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:40:09.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:09 smithi164 podman[61526]: 2024-04-26 15:40:09.330706288 +0000 UTC m=+0.420478000 container create 39e9ad09e5db39ce3566db357fd68cf4d8863f6136dd631b17e4d32dc84cb489 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-26T15:40:09.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 276317 9.79 MB/sec execute 573 sec latency 56.548 ms 2024-04-26T15:40:09.918 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:40:10.093 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:09 smithi164 podman[61526]: 2024-04-26 15:40:09.734052591 +0000 UTC m=+0.823824313 container init 39e9ad09e5db39ce3566db357fd68cf4d8863f6136dd631b17e4d32dc84cb489 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-26T15:40:10.093 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:09 smithi164 podman[61526]: 2024-04-26 15:40:09.737483015 +0000 UTC m=+0.827254727 container start 39e9ad09e5db39ce3566db357fd68cf4d8863f6136dd631b17e4d32dc84cb489 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True) 2024-04-26T15:40:10.093 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:09 smithi164 podman[61526]: 2024-04-26 15:40:09.833731536 +0000 UTC m=+0.923503248 container attach 39e9ad09e5db39ce3566db357fd68cf4d8863f6136dd631b17e4d32dc84cb489 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:40:10.276 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:10 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:10.277 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:10 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:40:10.277 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:10 smithi112 ceph-mon[63158]: pgmap v282: 97 pgs: 3 active+undersized, 18 active+undersized+degraded, 12 peering, 1 stale+active+clean, 63 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.4 MiB/s wr, 377 op/s; 130/1614 objects degraded (8.055%) 2024-04-26T15:40:10.277 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:10 smithi112 ceph-mon[63158]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:10.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:10.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:40:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-mon[49952]: pgmap v282: 97 pgs: 3 active+undersized, 18 active+undersized+degraded, 12 peering, 1 stale+active+clean, 63 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.4 MiB/s wr, 377 op/s; 130/1614 objects degraded (8.055%) 2024-04-26T15:40:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-mon[49952]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:10.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate[61541]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:40:10.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate[61541]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:40:10.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 bash[61526]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:40:10.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 bash[61526]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T15:40:10.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 bash[61526]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-26T15:40:10.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 bash[61526]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T15:40:10.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 bash[61526]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T15:40:10.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 bash[61526]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T15:40:10.349 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 bash[61526]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-26T15:40:10.349 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 bash[61526]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-26T15:40:10.349 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 bash[61526]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-26T15:40:10.349 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate[61541]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-26T15:40:10.349 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate[61541]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T15:40:10.349 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate[61541]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T15:40:10.349 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate[61541]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T15:40:10.349 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate[61541]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-26T15:40:10.349 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate[61541]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-26T15:40:10.349 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate[61541]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-26T15:40:10.349 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 podman[61526]: 2024-04-26 15:40:10.312534021 +0000 UTC m=+1.402305735 container died 39e9ad09e5db39ce3566db357fd68cf4d8863f6136dd631b17e4d32dc84cb489 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-26T15:40:10.865 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:40:10.865 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (9m) 2m ago 18m 25.1M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:40:10.865 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (5m) 2m ago 18m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:40:10.865 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (5m) 32s ago 17m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:40:10.865 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (8m) 2m ago 17m 54.1M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:40:10.865 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (12m) 2m ago 12m 38.3M - 17.2.0 e1d6a67b021e 956ba9e955c8 2024-04-26T15:40:10.865 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (12m) 32s ago 12m 15.0M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (8m) 2m ago 19m 522M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (7m) 32s ago 17m 468M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (7m) 2m ago 19m 52.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (6m) 32s ago 17m 44.6M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (11m) 2m ago 11m 266M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (11m) 32s ago 11m 90.2M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (8m) 2m ago 17m 17.5M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (8m) 32s ago 17m 18.4M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (5m) 2m ago 16m 353M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (4m) 2m ago 16m 291M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (3m) 2m ago 15m 213M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (2m) 2m ago 15m 14.9M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd 27fefb9c15b3 2024-04-26T15:40:10.866 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (113s) 32s ago 14m 297M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd b68352a89f2e 2024-04-26T15:40:10.867 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (76s) 32s ago 14m 278M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 61be8ee4a0ab 2024-04-26T15:40:10.867 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (37s) 32s ago 13m 12.4M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 78ef3c811a10 2024-04-26T15:40:10.867 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (13m) 32s ago 13m 593M 2704M 17.2.0 e1d6a67b021e ab696f7cadf9 2024-04-26T15:40:10.867 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (7m) 2m ago 16m 73.2M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:40:10.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 276644 9.78 MB/sec execute 574 sec latency 44.860 ms 2024-04-26T15:40:11.291 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:11 smithi112 ceph-mon[63158]: Health check failed: Degraded data redundancy: 130/1614 objects degraded (8.055%), 18 pgs degraded (PG_DEGRADED) 2024-04-26T15:40:11.291 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:11 smithi112 ceph-mon[63158]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:11.291 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:11 smithi112 ceph-mon[63158]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:11.335 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:11 smithi164 ceph-mon[49952]: Health check failed: Degraded data redundancy: 130/1614 objects degraded (8.055%), 18 pgs degraded (PG_DEGRADED) 2024-04-26T15:40:11.335 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:11 smithi164 ceph-mon[49952]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:11.335 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:11 smithi164 ceph-mon[49952]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:11.337 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:10 smithi164 podman[61663]: 2024-04-26 15:40:10.972058672 +0000 UTC m=+0.654442695 container remove 39e9ad09e5db39ce3566db357fd68cf4d8863f6136dd631b17e4d32dc84cb489 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-26T15:40:11.337 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:11 smithi164 podman[61729]: 2024-04-26 15:40:11.096130538 +0000 UTC m=+0.019628695 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:40:11.337 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:11 smithi164 podman[61729]: 2024-04-26 15:40:11.214861868 +0000 UTC m=+0.138360035 container create 955dfe6c53cec0539a3b6d8156bc7e09ce31b824cf277822406b46caa7f59c0f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-26T15:40:11.592 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:40:11.592 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:40:11.592 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:40:11.592 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:40:11.592 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:40:11.592 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:40:11.592 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:40:11.592 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:40:11.592 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 7 2024-04-26T15:40:11.592 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:40:11.592 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:40:11.593 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T15:40:11.593 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:40:11.593 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:40:11.593 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-26T15:40:11.593 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 11 2024-04-26T15:40:11.593 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:40:11.593 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:40:11.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:11 smithi164 podman[61729]: 2024-04-26 15:40:11.452189143 +0000 UTC m=+0.375687282 container init 955dfe6c53cec0539a3b6d8156bc7e09ce31b824cf277822406b46caa7f59c0f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-26T15:40:11.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:11 smithi164 podman[61729]: 2024-04-26 15:40:11.455896294 +0000 UTC m=+0.379394413 container start 955dfe6c53cec0539a3b6d8156bc7e09ce31b824cf277822406b46caa7f59c0f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-26T15:40:11.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:11 smithi164 ceph-osd[61748]: -- 172.21.15.164:0/3290265135 <== mon.1 v2:172.21.15.164:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a7e7e95680 con 0x55a7e7e5ec00 2024-04-26T15:40:11.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:11 smithi164 bash[61729]: 955dfe6c53cec0539a3b6d8156bc7e09ce31b824cf277822406b46caa7f59c0f 2024-04-26T15:40:11.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:11 smithi164 systemd[1]: Started Ceph osd.7 for 71e28ad8-03e0-11ef-bc93-c7b262605968. 2024-04-26T15:40:11.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 276939 9.77 MB/sec execute 575 sec latency 51.775 ms 2024-04-26T15:40:12.218 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:40:12.218 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:40:12.218 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:40:12.218 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:40:12.218 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:40:12.219 INFO:teuthology.orchestra.run.smithi112.stdout: "crash", 2024-04-26T15:40:12.219 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:40:12.219 INFO:teuthology.orchestra.run.smithi112.stdout: "mon" 2024-04-26T15:40:12.219 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:40:12.219 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "13/23 daemons upgraded", 2024-04-26T15:40:12.219 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T15:40:12.219 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:40:12.219 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:40:12.302 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:12 smithi112 ceph-mon[63158]: pgmap v283: 97 pgs: 5 active+undersized, 25 active+undersized+degraded, 12 peering, 55 active+clean; 354 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.3 MiB/s wr, 513 op/s; 183/1605 objects degraded (11.402%) 2024-04-26T15:40:12.302 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:12 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/2166263279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:12.302 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:12.302 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:12.302 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:40:12.357 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:11 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:40:11.998+0000 7f6ceecae7c0 -1 Falling back to public interface 2024-04-26T15:40:12.357 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:12 smithi164 ceph-mon[49952]: pgmap v283: 97 pgs: 5 active+undersized, 25 active+undersized+degraded, 12 peering, 55 active+clean; 354 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.3 MiB/s wr, 513 op/s; 183/1605 objects degraded (11.402%) 2024-04-26T15:40:12.357 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:12 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/2166263279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:12.357 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:12.357 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:12.358 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:40:12.820 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 183/1605 objects degraded (11.402%), 25 pgs degraded 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout: osd.7 (root=default,host=smithi164) is down 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 183/1605 objects degraded (11.402%), 25 pgs degraded 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-04-26T15:40:12.821 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.7 is active+undersized+degraded, acting [3,0] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.8 is active+undersized+degraded, acting [3,1] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.f is active+undersized+degraded, acting [4,0] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-04-26T15:40:12.822 INFO:teuthology.orchestra.run.smithi112.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-04-26T15:40:12.823 INFO:teuthology.orchestra.run.smithi112.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-04-26T15:40:12.823 INFO:teuthology.orchestra.run.smithi112.stdout: pg 4.14 is active+undersized+degraded, acting [3,1] 2024-04-26T15:40:12.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 277216 9.77 MB/sec execute 576 sec latency 53.157 ms 2024-04-26T15:40:13.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:13 smithi112 ceph-mon[63158]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:13.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:13 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/761600514' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:40:13.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:13 smithi164 ceph-mon[49952]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:13.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:13 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/761600514' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:40:13.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 277508 9.77 MB/sec execute 577 sec latency 64.040 ms 2024-04-26T15:40:14.191 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:14 smithi164 ceph-mon[49952]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s wr, 344 op/s; 260/1683 objects degraded (15.449%) 2024-04-26T15:40:14.362 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:14 smithi112 ceph-mon[63158]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s wr, 344 op/s; 260/1683 objects degraded (15.449%) 2024-04-26T15:40:14.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 277928 9.77 MB/sec execute 578 sec latency 58.854 ms 2024-04-26T15:40:15.386 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:15 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 260/1665 objects degraded (15.616%), 35 pgs degraded (PG_DEGRADED) 2024-04-26T15:40:15.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:15 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 260/1665 objects degraded (15.616%), 35 pgs degraded (PG_DEGRADED) 2024-04-26T15:40:15.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 278261 9.76 MB/sec execute 579 sec latency 38.481 ms 2024-04-26T15:40:16.405 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:16 smithi112 ceph-mon[63158]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 439 op/s; 260/1665 objects degraded (15.616%) 2024-04-26T15:40:16.405 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:16 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:16.406 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:16 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:16.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:16 smithi164 ceph-mon[49952]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 439 op/s; 260/1665 objects degraded (15.616%) 2024-04-26T15:40:16.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:16 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:16.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:16 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:16.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 278616 9.76 MB/sec execute 580 sec latency 38.973 ms 2024-04-26T15:40:17.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 279055 9.77 MB/sec execute 581 sec latency 45.915 ms 2024-04-26T15:40:18.681 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:18 smithi112 ceph-mon[63158]: pgmap v286: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 354 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 398 op/s; 258/1629 objects degraded (15.838%) 2024-04-26T15:40:18.682 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:18 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:18.682 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:18 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:18.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:18 smithi164 ceph-mon[49952]: pgmap v286: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 354 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 398 op/s; 258/1629 objects degraded (15.838%) 2024-04-26T15:40:18.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:18 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:18.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:18 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:18.877 INFO:teuthology.orchestra.run.smithi112.stdout: 5 279484 9.78 MB/sec execute 582 sec latency 42.209 ms 2024-04-26T15:40:19.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 279851 9.78 MB/sec execute 583 sec latency 52.615 ms 2024-04-26T15:40:20.735 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:20 smithi112 ceph-mon[63158]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 359 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 516 op/s; 253/1611 objects degraded (15.705%) 2024-04-26T15:40:20.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:20 smithi164 ceph-mon[49952]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 359 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 516 op/s; 253/1611 objects degraded (15.705%) 2024-04-26T15:40:20.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 280181 9.78 MB/sec execute 584 sec latency 38.603 ms 2024-04-26T15:40:21.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 280469 9.77 MB/sec execute 585 sec latency 61.404 ms 2024-04-26T15:40:22.098 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:21 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:40:21.843+0000 7f6ceecae7c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-26T15:40:22.340 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: pgmap v288: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 503 op/s; 253/1641 objects degraded (15.417%) 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: Upgrade: Setting container_image for all osd 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:22.341 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 253/1641 objects degraded (15.417%), 35 pgs degraded (PG_DEGRADED) 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-26T15:40:22.342 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-26T15:40:22.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-26T15:40:22.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-26T15:40:22.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: Upgrade: Setting require_osd_release to 19 squid 2024-04-26T15:40:22.343 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:22 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-26T15:40:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: pgmap v288: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 503 op/s; 253/1641 objects degraded (15.417%) 2024-04-26T15:40:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:40:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:40:22.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:22.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:22.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:40:22.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:22.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:22.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:22.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:22.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: Upgrade: Setting container_image for all osd 2024-04-26T15:40:22.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:22.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-26T15:40:22.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 253/1641 objects degraded (15.417%), 35 pgs degraded (PG_DEGRADED) 2024-04-26T15:40:22.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-26T15:40:22.601 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-26T15:40:22.601 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-26T15:40:22.601 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-26T15:40:22.601 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-26T15:40:22.601 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-26T15:40:22.601 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-26T15:40:22.601 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-26T15:40:22.601 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-26T15:40:22.601 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-26T15:40:22.601 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-26T15:40:22.601 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-26T15:40:22.602 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-26T15:40:22.602 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-26T15:40:22.602 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-26T15:40:22.602 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: Upgrade: Setting require_osd_release to 19 squid 2024-04-26T15:40:22.602 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:22 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-26T15:40:22.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 280754 9.77 MB/sec execute 586 sec latency 41.278 ms 2024-04-26T15:40:23.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:23 smithi112 ceph-mon[63158]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-26T15:40:23.576 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-26T15:40:23.576 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:23 smithi112 ceph-mon[63158]: osdmap e143: 8 total, 7 up, 8 in 2024-04-26T15:40:23.576 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi112.fqmway"]}]: dispatch 2024-04-26T15:40:23.576 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:23 smithi112 ceph-mon[63158]: Upgrade: It appears safe to stop mds.foofs.smithi112.fqmway 2024-04-26T15:40:23.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:23 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:40:23.539+0000 7f6ceecae7c0 -1 osd.7 140 log_to_monitors true 2024-04-26T15:40:23.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:23 smithi164 ceph-mon[49952]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-26T15:40:23.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-26T15:40:23.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:23 smithi164 ceph-mon[49952]: osdmap e143: 8 total, 7 up, 8 in 2024-04-26T15:40:23.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi112.fqmway"]}]: dispatch 2024-04-26T15:40:23.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:23 smithi164 ceph-mon[49952]: Upgrade: It appears safe to stop mds.foofs.smithi112.fqmway 2024-04-26T15:40:23.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 281010 9.77 MB/sec execute 587 sec latency 56.067 ms 2024-04-26T15:40:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:24 smithi164 ceph-mon[49952]: pgmap v290: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 585 op/s; 252/1653 objects degraded (15.245%) 2024-04-26T15:40:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:24 smithi164 ceph-mon[49952]: from='osd.7 [v2:172.21.15.164:6824/2516729574,v1:172.21.15.164:6825/2516729574]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:40:24.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:24 smithi164 ceph-mon[49952]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:40:24.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:24.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:40:24.602 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:24 smithi112 ceph-mon[63158]: pgmap v290: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 585 op/s; 252/1653 objects degraded (15.245%) 2024-04-26T15:40:24.602 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:24 smithi112 ceph-mon[63158]: from='osd.7 [v2:172.21.15.164:6824/2516729574,v1:172.21.15.164:6825/2516729574]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:40:24.602 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:24 smithi112 ceph-mon[63158]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:40:24.602 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:24.602 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:40:24.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 281207 9.76 MB/sec execute 588 sec latency 147.930 ms 2024-04-26T15:40:25.097 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:40:24 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:40:24.805+0000 7f6cdc0fe700 -1 osd.7 140 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:40:25.366 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:25 smithi164 ceph-mon[49952]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T15:40:25.366 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:25 smithi164 ceph-mon[49952]: osdmap e144: 8 total, 7 up, 8 in 2024-04-26T15:40:25.366 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:25 smithi164 ceph-mon[49952]: from='osd.7 [v2:172.21.15.164:6824/2516729574,v1:172.21.15.164:6825/2516729574]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:40:25.366 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:25 smithi164 ceph-mon[49952]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:40:25.367 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:25 smithi164 ceph-mon[49952]: from='osd.7 ' entity='osd.7' 2024-04-26T15:40:25.367 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:25 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:25.367 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:25 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.fqmway", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T15:40:25.367 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:25 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:40:25.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:25 smithi112 ceph-mon[63158]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T15:40:25.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:25 smithi112 ceph-mon[63158]: osdmap e144: 8 total, 7 up, 8 in 2024-04-26T15:40:25.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:25 smithi112 ceph-mon[63158]: from='osd.7 [v2:172.21.15.164:6824/2516729574,v1:172.21.15.164:6825/2516729574]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:40:25.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:25 smithi112 ceph-mon[63158]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-04-26T15:40:25.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:25 smithi112 ceph-mon[63158]: from='osd.7 ' entity='osd.7' 2024-04-26T15:40:25.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:25 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:25.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:25 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.fqmway", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T15:40:25.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:25 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:40:25.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 281504 9.76 MB/sec execute 589 sec latency 78.270 ms 2024-04-26T15:40:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:26 smithi112 ceph-mon[63158]: pgmap v292: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.6 MiB/s wr, 544 op/s; 249/1626 objects degraded (15.314%) 2024-04-26T15:40:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:26 smithi112 ceph-mon[63158]: Upgrade: Updating mds.foofs.smithi112.fqmway 2024-04-26T15:40:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:26 smithi112 ceph-mon[63158]: Deploying daemon mds.foofs.smithi112.fqmway on smithi112 2024-04-26T15:40:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:26 smithi112 ceph-mon[63158]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:40:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:26 smithi112 ceph-mon[63158]: osd.7 [v2:172.21.15.164:6824/2516729574,v1:172.21.15.164:6825/2516729574] boot 2024-04-26T15:40:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:26 smithi112 ceph-mon[63158]: osdmap e145: 8 total, 8 up, 8 in 2024-04-26T15:40:26.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:26 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:40:26.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:26 smithi164 ceph-mon[49952]: pgmap v292: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.6 MiB/s wr, 544 op/s; 249/1626 objects degraded (15.314%) 2024-04-26T15:40:26.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:26 smithi164 ceph-mon[49952]: Upgrade: Updating mds.foofs.smithi112.fqmway 2024-04-26T15:40:26.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:26 smithi164 ceph-mon[49952]: Deploying daemon mds.foofs.smithi112.fqmway on smithi112 2024-04-26T15:40:26.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:26 smithi164 ceph-mon[49952]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T15:40:26.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:26 smithi164 ceph-mon[49952]: osd.7 [v2:172.21.15.164:6824/2516729574,v1:172.21.15.164:6825/2516729574] boot 2024-04-26T15:40:26.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:26 smithi164 ceph-mon[49952]: osdmap e145: 8 total, 8 up, 8 in 2024-04-26T15:40:26.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:26 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:40:26.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 281546 9.74 MB/sec execute 590 sec latency 1042.313 ms 2024-04-26T15:40:27.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 281625 9.72 MB/sec execute 591 sec latency 2042.370 ms 2024-04-26T15:40:28.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:27 smithi164 ceph-mon[49952]: osdmap e146: 8 total, 8 up, 8 in 2024-04-26T15:40:28.098 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:40:27 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:40:27.996+0000 7f865e7c8700 -1 bdev(0x55e62a38e000 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 1 2024-04-26T15:40:28.098 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:40:27.999+0000 7f932c898700 -1 bdev(0x563ec4c28000 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 12 2024-04-26T15:40:28.098 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:40:28.001+0000 7f932d89a700 -1 bdev(0x563ec4c28000 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 14 2024-04-26T15:40:28.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:27 smithi112 ceph-mon[63158]: osdmap e146: 8 total, 8 up, 8 in 2024-04-26T15:40:28.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:27 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112[63131]: 2024-04-26T15:40:27.882+0000 7f046bbd9700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T15:40:28.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 281627 9.71 MB/sec execute 592 sec latency 2095.394 ms 2024-04-26T15:40:29.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-mon[49952]: pgmap v295: 97 pgs: 1 unknown, 8 peering, 5 active+undersized, 28 active+undersized+degraded, 55 active+clean; 357 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 3.4 MiB/s wr, 361 op/s; 213/1590 objects degraded (13.396%) 2024-04-26T15:40:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-mon[49952]: Health check update: Degraded data redundancy: 213/1590 objects degraded (13.396%), 28 pgs degraded (PG_DEGRADED) 2024-04-26T15:40:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-mon[49952]: osdmap e147: 8 total, 8 up, 8 in 2024-04-26T15:40:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-mon[49952]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-26T15:40:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-mon[49952]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T15:40:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-mon[49952]: osdmap e148: 8 total, 8 up, 8 in 2024-04-26T15:40:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-mon[49952]: Standby daemon mds.foofs.smithi164.gbxewm assigned to filesystem foofs as rank 0 2024-04-26T15:40:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-mon[49952]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-26T15:40:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-mon[49952]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T15:40:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-mon[49952]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-26T15:40:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:28 smithi164 ceph-mon[49952]: fsmap foofs:1/1 {0=foofs.smithi164.gbxewm=up:replay} 2024-04-26T15:40:29.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:28 smithi112 ceph-mon[63158]: pgmap v295: 97 pgs: 1 unknown, 8 peering, 5 active+undersized, 28 active+undersized+degraded, 55 active+clean; 357 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 3.4 MiB/s wr, 361 op/s; 213/1590 objects degraded (13.396%) 2024-04-26T15:40:29.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:28 smithi112 ceph-mon[63158]: Health check update: Degraded data redundancy: 213/1590 objects degraded (13.396%), 28 pgs degraded (PG_DEGRADED) 2024-04-26T15:40:29.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:28 smithi112 ceph-mon[63158]: osdmap e147: 8 total, 8 up, 8 in 2024-04-26T15:40:29.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:28 smithi112 ceph-mon[63158]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-26T15:40:29.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:28 smithi112 ceph-mon[63158]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T15:40:29.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:28 smithi112 ceph-mon[63158]: osdmap e148: 8 total, 8 up, 8 in 2024-04-26T15:40:29.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:28 smithi112 ceph-mon[63158]: Standby daemon mds.foofs.smithi164.gbxewm assigned to filesystem foofs as rank 0 2024-04-26T15:40:29.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:28 smithi112 ceph-mon[63158]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-26T15:40:29.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:28 smithi112 ceph-mon[63158]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T15:40:29.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:28 smithi112 ceph-mon[63158]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-26T15:40:29.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:28 smithi112 ceph-mon[63158]: fsmap foofs:1/1 {0=foofs.smithi164.gbxewm=up:replay} 2024-04-26T15:40:29.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 281627 9.69 MB/sec execute 593 sec latency 1947.062 ms 2024-04-26T15:40:30.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:29 smithi112 ceph-mon[63158]: osdmap e149: 8 total, 8 up, 8 in 2024-04-26T15:40:30.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:29 smithi112 ceph-mon[63158]: mds.? [v2:172.21.15.164:6832/3623172766,v1:172.21.15.164:6833/3623172766] up:reconnect 2024-04-26T15:40:30.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:29 smithi112 ceph-mon[63158]: fsmap foofs:1/1 {0=foofs.smithi164.gbxewm=up:reconnect} 2024-04-26T15:40:30.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:29 smithi164 ceph-mon[49952]: osdmap e149: 8 total, 8 up, 8 in 2024-04-26T15:40:30.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:29 smithi164 ceph-mon[49952]: mds.? [v2:172.21.15.164:6832/3623172766,v1:172.21.15.164:6833/3623172766] up:reconnect 2024-04-26T15:40:30.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:29 smithi164 ceph-mon[49952]: fsmap foofs:1/1 {0=foofs.smithi164.gbxewm=up:reconnect} 2024-04-26T15:40:30.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 281627 9.67 MB/sec execute 594 sec latency 2947.097 ms 2024-04-26T15:40:31.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:30 smithi112 ceph-mon[63158]: pgmap v299: 97 pgs: 1 activating+undersized+degraded+remapped, 3 remapped+peering, 1 unknown, 8 peering, 2 active+undersized, 1 active+undersized+remapped, 16 active+undersized+degraded, 65 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.6 MiB/s wr, 253 op/s; 144/1578 objects degraded (9.125%); 1.2 MiB/s, 12 objects/s recovering 2024-04-26T15:40:31.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:30 smithi112 ceph-mon[63158]: reconnect by client.24337 172.21.15.164:0/187023859 after 0.000999983 2024-04-26T15:40:31.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:30 smithi112 ceph-mon[63158]: reconnect by client.14546 172.21.15.112:0/2440813904 after 0.000999983 2024-04-26T15:40:31.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:30 smithi112 ceph-mon[63158]: osdmap e150: 8 total, 8 up, 8 in 2024-04-26T15:40:31.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:30 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:40:31.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:30 smithi112 ceph-mon[63158]: mds.? [v2:172.21.15.164:6832/3623172766,v1:172.21.15.164:6833/3623172766] up:rejoin 2024-04-26T15:40:31.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:30 smithi112 ceph-mon[63158]: fsmap foofs:1/1 {0=foofs.smithi164.gbxewm=up:rejoin} 2024-04-26T15:40:31.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:30 smithi112 ceph-mon[63158]: daemon mds.foofs.smithi164.gbxewm is now active in filesystem foofs as rank 0 2024-04-26T15:40:31.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:30 smithi164 ceph-mon[49952]: pgmap v299: 97 pgs: 1 activating+undersized+degraded+remapped, 3 remapped+peering, 1 unknown, 8 peering, 2 active+undersized, 1 active+undersized+remapped, 16 active+undersized+degraded, 65 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.6 MiB/s wr, 253 op/s; 144/1578 objects degraded (9.125%); 1.2 MiB/s, 12 objects/s recovering 2024-04-26T15:40:31.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:30 smithi164 ceph-mon[49952]: reconnect by client.24337 172.21.15.164:0/187023859 after 0.000999983 2024-04-26T15:40:31.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:30 smithi164 ceph-mon[49952]: reconnect by client.14546 172.21.15.112:0/2440813904 after 0.000999983 2024-04-26T15:40:31.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:30 smithi164 ceph-mon[49952]: osdmap e150: 8 total, 8 up, 8 in 2024-04-26T15:40:31.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:30 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:40:31.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:30 smithi164 ceph-mon[49952]: mds.? [v2:172.21.15.164:6832/3623172766,v1:172.21.15.164:6833/3623172766] up:rejoin 2024-04-26T15:40:31.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:30 smithi164 ceph-mon[49952]: fsmap foofs:1/1 {0=foofs.smithi164.gbxewm=up:rejoin} 2024-04-26T15:40:31.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:30 smithi164 ceph-mon[49952]: daemon mds.foofs.smithi164.gbxewm is now active in filesystem foofs as rank 0 2024-04-26T15:40:31.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 281627 9.66 MB/sec execute 595 sec latency 3947.132 ms 2024-04-26T15:40:32.139 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:31 smithi112 ceph-mon[63158]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-26T15:40:32.139 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:31 smithi112 ceph-mon[63158]: mds.? [v2:172.21.15.164:6832/3623172766,v1:172.21.15.164:6833/3623172766] up:active 2024-04-26T15:40:32.140 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:31 smithi112 ceph-mon[63158]: fsmap foofs:1 {0=foofs.smithi164.gbxewm=up:active} 2024-04-26T15:40:32.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:31 smithi164 ceph-mon[49952]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-26T15:40:32.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:31 smithi164 ceph-mon[49952]: mds.? [v2:172.21.15.164:6832/3623172766,v1:172.21.15.164:6833/3623172766] up:active 2024-04-26T15:40:32.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:31 smithi164 ceph-mon[49952]: fsmap foofs:1 {0=foofs.smithi164.gbxewm=up:active} 2024-04-26T15:40:32.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 281952 9.65 MB/sec execute 596 sec latency 3955.291 ms 2024-04-26T15:40:33.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:32 smithi112 ceph-mon[63158]: pgmap v301: 97 pgs: 1 activating+undersized+degraded+remapped, 2 remapped+peering, 1 unknown, 15 peering, 78 active+clean; 350 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 1.8 MiB/s wr, 177 op/s; 22/1569 objects degraded (1.402%); 1.1 MiB/s, 24 objects/s recovering 2024-04-26T15:40:33.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:32 smithi164 ceph-mon[49952]: pgmap v301: 97 pgs: 1 activating+undersized+degraded+remapped, 2 remapped+peering, 1 unknown, 15 peering, 78 active+clean; 350 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 1.8 MiB/s wr, 177 op/s; 22/1569 objects degraded (1.402%); 1.1 MiB/s, 24 objects/s recovering 2024-04-26T15:40:33.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 282259 9.65 MB/sec execute 597 sec latency 39.744 ms 2024-04-26T15:40:34.168 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:33 smithi112 ceph-mon[63158]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-26T15:40:34.168 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:33 smithi112 ceph-mon[63158]: mds.? [v2:172.21.15.112:6834/2034933266,v1:172.21.15.112:6835/2034933266] up:boot 2024-04-26T15:40:34.168 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:33 smithi112 ceph-mon[63158]: fsmap foofs:1 {0=foofs.smithi164.gbxewm=up:active} 1 up:standby 2024-04-26T15:40:34.169 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:33 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:40:34.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:33 smithi164 ceph-mon[49952]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-26T15:40:34.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:33 smithi164 ceph-mon[49952]: mds.? [v2:172.21.15.112:6834/2034933266,v1:172.21.15.112:6835/2034933266] up:boot 2024-04-26T15:40:34.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:33 smithi164 ceph-mon[49952]: fsmap foofs:1 {0=foofs.smithi164.gbxewm=up:active} 1 up:standby 2024-04-26T15:40:34.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:33 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:40:34.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 282530 9.65 MB/sec execute 598 sec latency 64.460 ms 2024-04-26T15:40:34.939 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:34 smithi112 ceph-mon[63158]: pgmap v302: 97 pgs: 7 peering, 90 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 1.3 MiB/s wr, 151 op/s; 2.5 MiB/s, 29 objects/s recovering 2024-04-26T15:40:34.939 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:34 smithi112 ceph-mon[63158]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/1569 objects degraded (1.402%), 1 pg degraded) 2024-04-26T15:40:34.939 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:34 smithi112 ceph-mon[63158]: Cluster is now healthy 2024-04-26T15:40:35.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:34 smithi164 ceph-mon[49952]: pgmap v302: 97 pgs: 7 peering, 90 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 1.3 MiB/s wr, 151 op/s; 2.5 MiB/s, 29 objects/s recovering 2024-04-26T15:40:35.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:34 smithi164 ceph-mon[49952]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/1569 objects degraded (1.402%), 1 pg degraded) 2024-04-26T15:40:35.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:34 smithi164 ceph-mon[49952]: Cluster is now healthy 2024-04-26T15:40:35.881 INFO:teuthology.orchestra.run.smithi112.stdout: 5 282771 9.65 MB/sec execute 599 sec latency 82.907 ms 2024-04-26T15:40:36.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 cleanup 600 sec 2024-04-26T15:40:36.990 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:36 smithi112 ceph-mon[63158]: pgmap v303: 97 pgs: 97 active+clean; 351 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 1.4 MiB/s wr, 215 op/s; 2.8 MiB/s, 33 objects/s recovering 2024-04-26T15:40:37.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:36 smithi164 ceph-mon[49952]: pgmap v303: 97 pgs: 97 active+clean; 351 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 1.4 MiB/s wr, 215 op/s; 2.8 MiB/s, 33 objects/s recovering 2024-04-26T15:40:37.878 INFO:teuthology.orchestra.run.smithi112.stdout: 5 cleanup 601 sec 2024-04-26T15:40:38.420 INFO:teuthology.orchestra.run.smithi112.stdout: 0 cleanup 602 sec 2024-04-26T15:40:38.420 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:40:38.420 INFO:teuthology.orchestra.run.smithi112.stdout: Operation Count AvgLat MaxLat 2024-04-26T15:40:38.420 INFO:teuthology.orchestra.run.smithi112.stdout: ---------------------------------------- 2024-04-26T15:40:38.420 INFO:teuthology.orchestra.run.smithi112.stdout: NTCreateX 185611 4.653 2435.140 2024-04-26T15:40:38.420 INFO:teuthology.orchestra.run.smithi112.stdout: Close 136428 0.132 7.912 2024-04-26T15:40:38.420 INFO:teuthology.orchestra.run.smithi112.stdout: Rename 7871 9.416 163.178 2024-04-26T15:40:38.421 INFO:teuthology.orchestra.run.smithi112.stdout: Unlink 37402 7.057 2286.252 2024-04-26T15:40:38.421 INFO:teuthology.orchestra.run.smithi112.stdout: Qpathinfo 168480 1.177 2273.427 2024-04-26T15:40:38.421 INFO:teuthology.orchestra.run.smithi112.stdout: Qfileinfo 29405 0.006 1.301 2024-04-26T15:40:38.421 INFO:teuthology.orchestra.run.smithi112.stdout: Qfsinfo 30802 0.579 74.463 2024-04-26T15:40:38.421 INFO:teuthology.orchestra.run.smithi112.stdout: Sfileinfo 15153 13.030 2357.603 2024-04-26T15:40:38.421 INFO:teuthology.orchestra.run.smithi112.stdout: Find 65048 0.922 9.732 2024-04-26T15:40:38.421 INFO:teuthology.orchestra.run.smithi112.stdout: WriteX 91764 14.162 3955.286 2024-04-26T15:40:38.421 INFO:teuthology.orchestra.run.smithi112.stdout: ReadX 291383 0.010 7.204 2024-04-26T15:40:38.421 INFO:teuthology.orchestra.run.smithi112.stdout: LockX 606 0.419 10.200 2024-04-26T15:40:38.421 INFO:teuthology.orchestra.run.smithi112.stdout: UnlockX 606 0.357 10.940 2024-04-26T15:40:38.421 INFO:teuthology.orchestra.run.smithi112.stdout: Flush 13005 0.002 0.042 2024-04-26T15:40:38.422 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:40:38.422 INFO:teuthology.orchestra.run.smithi112.stdout:Throughput 9.64605 MB/sec 5 clients 5 procs max_latency=3955.291 ms 2024-04-26T15:40:38.430 DEBUG:teuthology.orchestra.run.smithi112:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-04-26T15:40:38.481 DEBUG:teuthology.orchestra.run.smithi112:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-04-26T15:40:38.623 DEBUG:teuthology.orchestra.run.smithi112:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-04-26T15:40:38.665 INFO:teuthology.orchestra.run.smithi112.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-26T15:40:38.665 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:40:38.665 INFO:teuthology.orchestra.run.smithi112.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-04-26T15:40:38.666 INFO:teuthology.orchestra.run.smithi112.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-26T15:40:38.766 INFO:teuthology.orchestra.run.smithi112.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-26T15:40:38.766 INFO:teuthology.orchestra.run.smithi112.stdout:releasing clients 2024-04-26T15:40:39.133 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:38 smithi112 ceph-mon[63158]: pgmap v304: 97 pgs: 97 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.7 MiB/s wr, 209 op/s; 1.9 MiB/s, 29 objects/s recovering 2024-04-26T15:40:39.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:38 smithi164 ceph-mon[49952]: pgmap v304: 97 pgs: 97 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.7 MiB/s wr, 209 op/s; 1.9 MiB/s, 29 objects/s recovering 2024-04-26T15:40:40.203 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:40.204 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:40:40.204 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:40 smithi112 ceph-mon[63158]: pgmap v305: 97 pgs: 97 active+clean; 358 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.3 MiB/s wr, 272 op/s; 1.6 MiB/s, 24 objects/s recovering 2024-04-26T15:40:40.204 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:40 smithi112 ceph-mon[63158]: daemon mds.foofs.smithi112.fqmway restarted 2024-04-26T15:40:40.204 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:40.204 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:40 smithi112 ceph-mon[63158]: osdmap e151: 8 total, 8 up, 8 in 2024-04-26T15:40:40.204 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:40.204 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:40:40.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:40.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:40:40.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:40 smithi164 ceph-mon[49952]: pgmap v305: 97 pgs: 97 active+clean; 358 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.3 MiB/s wr, 272 op/s; 1.6 MiB/s, 24 objects/s recovering 2024-04-26T15:40:40.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:40 smithi164 ceph-mon[49952]: daemon mds.foofs.smithi112.fqmway restarted 2024-04-26T15:40:40.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:40.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:40 smithi164 ceph-mon[49952]: osdmap e151: 8 total, 8 up, 8 in 2024-04-26T15:40:40.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:40.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:40:40.766 INFO:teuthology.orchestra.run.smithi112.stdout: 5 242 15.09 MB/sec execute 1 sec latency 143.748 ms 2024-04-26T15:40:41.277 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:41 smithi112 ceph-mon[63158]: mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574] up:boot 2024-04-26T15:40:41.277 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:41 smithi112 ceph-mon[63158]: fsmap foofs:1 {0=foofs.smithi164.gbxewm=up:active} 1 up:standby 2024-04-26T15:40:41.277 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:41 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:40:41.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:41 smithi164 ceph-mon[49952]: mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574] up:boot 2024-04-26T15:40:41.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:41 smithi164 ceph-mon[49952]: fsmap foofs:1 {0=foofs.smithi164.gbxewm=up:active} 1 up:standby 2024-04-26T15:40:41.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:41 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:40:41.766 INFO:teuthology.orchestra.run.smithi112.stdout: 5 322 13.78 MB/sec execute 2 sec latency 99.736 ms 2024-04-26T15:40:42.334 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:42 smithi112 ceph-mon[63158]: pgmap v307: 97 pgs: 97 active+clean; 365 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.8 MiB/s wr, 242 op/s; 1.6 MiB/s, 18 objects/s recovering 2024-04-26T15:40:42.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:42 smithi164 ceph-mon[49952]: pgmap v307: 97 pgs: 97 active+clean; 365 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.8 MiB/s wr, 242 op/s; 1.6 MiB/s, 18 objects/s recovering 2024-04-26T15:40:42.766 INFO:teuthology.orchestra.run.smithi112.stdout: 5 410 13.78 MB/sec execute 3 sec latency 105.344 ms 2024-04-26T15:40:43.267 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:40:43.766 INFO:teuthology.orchestra.run.smithi112.stdout: 5 521 14.81 MB/sec execute 4 sec latency 80.417 ms 2024-04-26T15:40:44.115 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:40:44.115 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (9m) 2m ago 18m 25.1M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:40:44.115 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (6m) 2m ago 18m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:40:44.115 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (6m) 27s ago 18m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:40:44.115 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (9m) 2m ago 18m 54.1M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:40:44.115 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 starting - - - - 2024-04-26T15:40:44.115 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (12m) 27s ago 12m 15.1M - 17.2.0 e1d6a67b021e b5967d8ec06c 2024-04-26T15:40:44.115 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (9m) 2m ago 19m 522M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:40:44.115 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (8m) 27s ago 17m 468M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:40:44.115 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (7m) 2m ago 20m 52.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (7m) 27s ago 17m 45.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (12m) 2m ago 12m 266M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (12m) 27s ago 12m 90.2M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (9m) 2m ago 18m 17.5M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (9m) 27s ago 17m 18.5M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (5m) 2m ago 17m 353M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (4m) 2m ago 16m 291M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (3m) 2m ago 16m 213M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (3m) 2m ago 15m 14.9M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd 27fefb9c15b3 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (2m) 27s ago 15m 319M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd b68352a89f2e 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (109s) 27s ago 14m 309M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 61be8ee4a0ab 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (71s) 27s ago 14m 288M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 78ef3c811a10 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (33s) 27s ago 14m 12.2M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 955dfe6c53ce 2024-04-26T15:40:44.116 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (8m) 2m ago 17m 73.2M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:40:44.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:44 smithi112 ceph-mon[63158]: pgmap v308: 97 pgs: 97 active+clean; 339 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 447 KiB/s rd, 6.2 MiB/s wr, 273 op/s; 692 KiB/s, 11 objects/s recovering 2024-04-26T15:40:44.427 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:44 smithi112 ceph-mon[63158]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:44.428 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:44 smithi112 ceph-mon[63158]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:44.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:44 smithi164 ceph-mon[49952]: pgmap v308: 97 pgs: 97 active+clean; 339 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 447 KiB/s rd, 6.2 MiB/s wr, 273 op/s; 692 KiB/s, 11 objects/s recovering 2024-04-26T15:40:44.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:44 smithi164 ceph-mon[49952]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:44.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:44 smithi164 ceph-mon[49952]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:44.626 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 1 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:40:44.627 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:40:44.628 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-26T15:40:44.628 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 13 2024-04-26T15:40:44.628 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:40:44.628 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:40:44.766 INFO:teuthology.orchestra.run.smithi112.stdout: 5 cleanup 5 sec 2024-04-26T15:40:45.076 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:40:45.076 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:40:45.076 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:40:45.077 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:40:45.077 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:40:45.077 INFO:teuthology.orchestra.run.smithi112.stdout: "crash", 2024-04-26T15:40:45.077 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:40:45.077 INFO:teuthology.orchestra.run.smithi112.stdout: "mon", 2024-04-26T15:40:45.078 INFO:teuthology.orchestra.run.smithi112.stdout: "osd" 2024-04-26T15:40:45.078 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:40:45.078 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "14/23 daemons upgraded", 2024-04-26T15:40:45.078 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading mds daemons", 2024-04-26T15:40:45.078 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:40:45.078 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:40:45.274 INFO:teuthology.orchestra.run.smithi112.stdout: 0 cleanup 6 sec 2024-04-26T15:40:45.274 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: Operation Count AvgLat MaxLat 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: ---------------------------------------- 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: NTCreateX 342 35.462 105.341 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: Close 329 15.815 97.136 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: Rename 3 36.129 41.895 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: Unlink 8 12.343 28.860 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: Qpathinfo 32 4.110 26.999 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: Qfileinfo 315 19.762 61.111 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: Qfsinfo 39 0.408 1.945 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: Sfileinfo 19 20.826 28.882 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: Find 36 0.769 1.756 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: WriteX 1498 0.028 0.082 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: ReadX 131 0.012 0.203 2024-04-26T15:40:45.275 INFO:teuthology.orchestra.run.smithi112.stdout: Flush 34 21.136 31.995 2024-04-26T15:40:45.276 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:40:45.276 INFO:teuthology.orchestra.run.smithi112.stdout:Throughput 14.8128 MB/sec 5 clients 5 procs max_latency=143.748 ms 2024-04-26T15:40:45.277 DEBUG:teuthology.parallel:result is None 2024-04-26T15:40:45.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:45 smithi112 ceph-mon[63158]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:45.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:45.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:45 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/2872109245' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:45.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:45 smithi112 ceph-mon[63158]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:45.532 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:40:45.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:45 smithi164 ceph-mon[49952]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:45.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:45.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:45 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/2872109245' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:45.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:45 smithi164 ceph-mon[49952]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:40:46.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:46 smithi112 ceph-mon[63158]: pgmap v309: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 9.4 MiB/s wr, 336 op/s 2024-04-26T15:40:46.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:46 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/133247889' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:40:46.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:46 smithi164 ceph-mon[49952]: pgmap v309: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 9.4 MiB/s wr, 336 op/s 2024-04-26T15:40:46.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:46 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/133247889' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:40:48.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:48 smithi164 ceph-mon[49952]: pgmap v310: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 11 MiB/s wr, 325 op/s 2024-04-26T15:40:48.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:48 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:48.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:48 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:48.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:48 smithi112 ceph-mon[63158]: pgmap v310: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 11 MiB/s wr, 325 op/s 2024-04-26T15:40:48.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:48 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:48.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:48 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:49.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:49 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:49.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:49 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:49.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:49 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:49.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:49 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:50.607 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:50 smithi112 ceph-mon[63158]: pgmap v311: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 11 MiB/s wr, 238 op/s 2024-04-26T15:40:50.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:50 smithi164 ceph-mon[49952]: pgmap v311: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 11 MiB/s wr, 238 op/s 2024-04-26T15:40:52.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:52 smithi112 ceph-mon[63158]: pgmap v312: 97 pgs: 97 active+clean; 303 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 224 op/s 2024-04-26T15:40:52.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:52 smithi164 ceph-mon[49952]: pgmap v312: 97 pgs: 97 active+clean; 303 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 224 op/s 2024-04-26T15:40:53.517 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: Detected new or changed devices on smithi112 2024-04-26T15:40:53.517 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:53.517 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:53.517 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:40:53.517 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:40:53.517 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:53.518 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:53.518 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:40:53.518 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:53.518 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:53.518 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:53.518 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:53.518 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi164.gbxewm"]}]: dispatch 2024-04-26T15:40:53.518 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:53 smithi164 ceph-mon[49952]: Upgrade: It appears safe to stop mds.foofs.smithi164.gbxewm 2024-04-26T15:40:53.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: Detected new or changed devices on smithi112 2024-04-26T15:40:53.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:53.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:53.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:40:53.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:40:53.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:53.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:53.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:40:53.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:53.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:53.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:53.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:40:53.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi164.gbxewm"]}]: dispatch 2024-04-26T15:40:53.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:53 smithi112 ceph-mon[63158]: Upgrade: It appears safe to stop mds.foofs.smithi164.gbxewm 2024-04-26T15:40:54.761 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:54 smithi164 ceph-mon[49952]: pgmap v313: 97 pgs: 97 active+clean; 252 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.4 MiB/s wr, 219 op/s 2024-04-26T15:40:54.761 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:40:54.761 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:54.761 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.gbxewm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T15:40:54.761 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:40:54.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:54 smithi112 ceph-mon[63158]: pgmap v313: 97 pgs: 97 active+clean; 252 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.4 MiB/s wr, 219 op/s 2024-04-26T15:40:54.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:40:54.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:40:54.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.gbxewm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T15:40:54.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:40:55.693 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:55 smithi164 ceph-mon[49952]: Upgrade: Updating mds.foofs.smithi164.gbxewm 2024-04-26T15:40:55.693 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:55 smithi164 ceph-mon[49952]: Deploying daemon mds.foofs.smithi164.gbxewm on smithi164 2024-04-26T15:40:55.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:55 smithi112 ceph-mon[63158]: Upgrade: Updating mds.foofs.smithi164.gbxewm 2024-04-26T15:40:55.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:55 smithi112 ceph-mon[63158]: Deploying daemon mds.foofs.smithi164.gbxewm on smithi164 2024-04-26T15:40:56.509 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:56 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112[63131]: 2024-04-26T15:40:56.231+0000 7f046bbd9700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T15:40:56.509 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:40:56 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[71874]: 2024-04-26T15:40:56.263+0000 7f1393c68700 -1 bdev(0x55bc13ca0000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 1 2024-04-26T15:40:56.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:56 smithi164 ceph-mon[49952]: pgmap v314: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 182 op/s 2024-04-26T15:40:56.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:56 smithi164 ceph-mon[49952]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-26T15:40:56.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:56 smithi164 ceph-mon[49952]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T15:40:56.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:56 smithi164 ceph-mon[49952]: osdmap e152: 8 total, 8 up, 8 in 2024-04-26T15:40:56.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:56 smithi164 ceph-mon[49952]: Standby daemon mds.foofs.smithi112.fqmway assigned to filesystem foofs as rank 0 2024-04-26T15:40:56.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:56 smithi164 ceph-mon[49952]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-26T15:40:56.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:56 smithi164 ceph-mon[49952]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T15:40:56.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:56 smithi164 ceph-mon[49952]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-26T15:40:56.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:56 smithi164 ceph-mon[49952]: fsmap foofs:1/1 {0=foofs.smithi112.fqmway=up:replay} 2024-04-26T15:40:56.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:56 smithi112 ceph-mon[63158]: pgmap v314: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 182 op/s 2024-04-26T15:40:56.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:56 smithi112 ceph-mon[63158]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-26T15:40:56.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:56 smithi112 ceph-mon[63158]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T15:40:56.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:56 smithi112 ceph-mon[63158]: osdmap e152: 8 total, 8 up, 8 in 2024-04-26T15:40:56.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:56 smithi112 ceph-mon[63158]: Standby daemon mds.foofs.smithi112.fqmway assigned to filesystem foofs as rank 0 2024-04-26T15:40:56.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:56 smithi112 ceph-mon[63158]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-26T15:40:56.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:56 smithi112 ceph-mon[63158]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T15:40:56.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:56 smithi112 ceph-mon[63158]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-26T15:40:56.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:56 smithi112 ceph-mon[63158]: fsmap foofs:1/1 {0=foofs.smithi112.fqmway=up:replay} 2024-04-26T15:40:58.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:58 smithi164 ceph-mon[49952]: pgmap v316: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 823 KiB/s rd, 1.4 MiB/s wr, 60 op/s 2024-04-26T15:40:58.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:58 smithi164 ceph-mon[49952]: reconnect by client.14546 172.21.15.112:0/2440813904 after 0.000999974 2024-04-26T15:40:58.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:58 smithi164 ceph-mon[49952]: mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574] up:reconnect 2024-04-26T15:40:58.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:58 smithi164 ceph-mon[49952]: fsmap foofs:1/1 {0=foofs.smithi112.fqmway=up:reconnect} 2024-04-26T15:40:58.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:58 smithi164 ceph-mon[49952]: reconnect by client.24337 172.21.15.164:0/187023859 after 0.00799979 2024-04-26T15:40:58.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:58 smithi112 ceph-mon[63158]: pgmap v316: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 823 KiB/s rd, 1.4 MiB/s wr, 60 op/s 2024-04-26T15:40:58.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:58 smithi112 ceph-mon[63158]: reconnect by client.14546 172.21.15.112:0/2440813904 after 0.000999974 2024-04-26T15:40:58.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:58 smithi112 ceph-mon[63158]: mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574] up:reconnect 2024-04-26T15:40:58.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:58 smithi112 ceph-mon[63158]: fsmap foofs:1/1 {0=foofs.smithi112.fqmway=up:reconnect} 2024-04-26T15:40:58.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:58 smithi112 ceph-mon[63158]: reconnect by client.24337 172.21.15.164:0/187023859 after 0.00799979 2024-04-26T15:40:59.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:59 smithi164 ceph-mon[49952]: mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574] up:rejoin 2024-04-26T15:40:59.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:59 smithi164 ceph-mon[49952]: fsmap foofs:1/1 {0=foofs.smithi112.fqmway=up:rejoin} 2024-04-26T15:40:59.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:59 smithi164 ceph-mon[49952]: daemon mds.foofs.smithi112.fqmway is now active in filesystem foofs as rank 0 2024-04-26T15:40:59.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:59 smithi164 ceph-mon[49952]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-26T15:40:59.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:59 smithi164 ceph-mon[49952]: mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574] up:active 2024-04-26T15:40:59.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:40:59 smithi164 ceph-mon[49952]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 2024-04-26T15:40:59.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:59 smithi112 ceph-mon[63158]: mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574] up:rejoin 2024-04-26T15:40:59.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:59 smithi112 ceph-mon[63158]: fsmap foofs:1/1 {0=foofs.smithi112.fqmway=up:rejoin} 2024-04-26T15:40:59.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:59 smithi112 ceph-mon[63158]: daemon mds.foofs.smithi112.fqmway is now active in filesystem foofs as rank 0 2024-04-26T15:40:59.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:59 smithi112 ceph-mon[63158]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-26T15:40:59.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:59 smithi112 ceph-mon[63158]: mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574] up:active 2024-04-26T15:40:59.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:40:59 smithi112 ceph-mon[63158]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 2024-04-26T15:41:00.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:00 smithi164 ceph-mon[49952]: pgmap v317: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 443 KiB/s wr, 50 op/s 2024-04-26T15:41:00.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:00 smithi112 ceph-mon[63158]: pgmap v317: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 443 KiB/s wr, 50 op/s 2024-04-26T15:41:02.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:02 smithi164 ceph-mon[49952]: pgmap v318: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 13 KiB/s wr, 23 op/s 2024-04-26T15:41:02.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:02 smithi112 ceph-mon[63158]: pgmap v318: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 13 KiB/s wr, 23 op/s 2024-04-26T15:41:04.703 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:04 smithi164 ceph-mon[49952]: pgmap v319: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 13 KiB/s wr, 12 op/s 2024-04-26T15:41:04.703 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:04 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:04.703 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:04 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:04.704 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:04 smithi164 ceph-mon[49952]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-26T15:41:04.704 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:04 smithi164 ceph-mon[49952]: Cluster is now healthy 2024-04-26T15:41:04.704 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:04 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:41:04.704 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:04 smithi164 ceph-mon[49952]: mds.? [v2:172.21.15.164:6832/1374862397,v1:172.21.15.164:6833/1374862397] up:boot 2024-04-26T15:41:04.704 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:04 smithi164 ceph-mon[49952]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 1 up:standby 2024-04-26T15:41:04.704 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:04 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.gbxewm"}]: dispatch 2024-04-26T15:41:04.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:04 smithi112 ceph-mon[63158]: pgmap v319: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 13 KiB/s wr, 12 op/s 2024-04-26T15:41:04.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:04 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:04.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:04 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:04.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:04 smithi112 ceph-mon[63158]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-26T15:41:04.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:04 smithi112 ceph-mon[63158]: Cluster is now healthy 2024-04-26T15:41:04.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:04 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:41:04.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:04 smithi112 ceph-mon[63158]: mds.? [v2:172.21.15.164:6832/1374862397,v1:172.21.15.164:6833/1374862397] up:boot 2024-04-26T15:41:04.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:04 smithi112 ceph-mon[63158]: fsmap foofs:1 {0=foofs.smithi112.fqmway=up:active} 1 up:standby 2024-04-26T15:41:04.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:04 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.gbxewm"}]: dispatch 2024-04-26T15:41:06.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:06 smithi164 ceph-mon[49952]: pgmap v320: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 307 B/s wr, 11 op/s 2024-04-26T15:41:06.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:06 smithi112 ceph-mon[63158]: pgmap v320: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 307 B/s wr, 11 op/s 2024-04-26T15:41:08.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:08 smithi112 ceph-mon[63158]: pgmap v321: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 188 B/s wr, 10 op/s 2024-04-26T15:41:08.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:08.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:08.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:08 smithi164 ceph-mon[49952]: pgmap v321: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 188 B/s wr, 10 op/s 2024-04-26T15:41:08.754 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:08.754 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:09.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:09.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:09.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:41:09.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:09.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:09.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:41:10.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:10 smithi164 ceph-mon[49952]: pgmap v322: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 170 B/s wr, 9 op/s 2024-04-26T15:41:10.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:10 smithi112 ceph-mon[63158]: pgmap v322: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 170 B/s wr, 9 op/s 2024-04-26T15:41:12.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: pgmap v323: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 6.8 KiB/s wr, 10 op/s 2024-04-26T15:41:12.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: Detected new or changed devices on smithi164 2024-04-26T15:41:12.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:12.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:41:12.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:41:12.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.641 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: Upgrade: Setting container_image for all mds 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi112.fqmway"}]': finished 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi164.gbxewm"}]: dispatch 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi164.gbxewm"}]': finished 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: Upgrade: Setting container_image for all rgw 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.643 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.643 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T15:41:12.643 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.643 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.643 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: Upgrade: Setting container_image for all iscsi 2024-04-26T15:41:12.643 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:12 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: pgmap v323: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 6.8 KiB/s wr, 10 op/s 2024-04-26T15:41:12.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: Detected new or changed devices on smithi164 2024-04-26T15:41:12.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:12.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:41:12.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:41:12.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: Upgrade: Setting container_image for all mds 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi112.fqmway"}]: dispatch 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi112.fqmway"}]': finished 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi164.gbxewm"}]: dispatch 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi164.gbxewm"}]': finished 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: Upgrade: Setting container_image for all rgw 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T15:41:12.849 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T15:41:12.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:12.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:12.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: Upgrade: Setting container_image for all iscsi 2024-04-26T15:41:12.850 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:12 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:14.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: pgmap v324: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 6.7 KiB/s wr, 6 op/s 2024-04-26T15:41:14.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: Upgrade: Updating nfs.foo.0.0.smithi112.qswmgy (1/2) 2024-04-26T15:41:14.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:14.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: Creating key for client.nfs.foo.0.0.smithi112.qswmgy 2024-04-26T15:41:14.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi112.qswmgy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-26T15:41:14.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-26T15:41:14.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-26T15:41:14.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-26T15:41:14.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:14.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-26T15:41:14.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-26T15:41:14.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: Rados config object exists: conf-nfs.foo 2024-04-26T15:41:14.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: Creating key for client.nfs.foo.0.0.smithi112.qswmgy-rgw 2024-04-26T15:41:14.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi112.qswmgy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:41:14.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: Bind address in nfs.foo.0.0.smithi112.qswmgy's ganesha conf is defaulting to empty 2024-04-26T15:41:14.712 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:14.713 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:14 smithi112 ceph-mon[63158]: Deploying daemon nfs.foo.0.0.smithi112.qswmgy on smithi112 2024-04-26T15:41:14.746 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: pgmap v324: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 6.7 KiB/s wr, 6 op/s 2024-04-26T15:41:14.746 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: Upgrade: Updating nfs.foo.0.0.smithi112.qswmgy (1/2) 2024-04-26T15:41:14.746 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:14.746 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: Creating key for client.nfs.foo.0.0.smithi112.qswmgy 2024-04-26T15:41:14.746 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi112.qswmgy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-26T15:41:14.746 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-26T15:41:14.746 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-26T15:41:14.746 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-26T15:41:14.746 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:14.746 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-26T15:41:14.746 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-26T15:41:14.747 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: Rados config object exists: conf-nfs.foo 2024-04-26T15:41:14.747 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: Creating key for client.nfs.foo.0.0.smithi112.qswmgy-rgw 2024-04-26T15:41:14.747 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi112.qswmgy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:41:14.747 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: Bind address in nfs.foo.0.0.smithi112.qswmgy's ganesha conf is defaulting to empty 2024-04-26T15:41:14.747 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:14.747 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:14 smithi164 ceph-mon[49952]: Deploying daemon nfs.foo.0.0.smithi112.qswmgy on smithi112 2024-04-26T15:41:15.972 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:41:16.715 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:41:16.715 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (10m) 29s ago 19m 25.5M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:41:16.715 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (6m) 29s ago 19m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:41:16.715 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (6m) 9s ago 18m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (9m) 29s ago 18m 54.2M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (37s) 29s ago 13m 12.2M - 19.0.0-3305-ga430f9ed 031ba2747cfd 3b013d074da0 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (13s) 9s ago 13m 15.0M - 19.0.0-3305-ga430f9ed 031ba2747cfd fb6f8c982ad9 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (9m) 29s ago 20m 527M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (8m) 9s ago 18m 469M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (8m) 29s ago 20m 60.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (7m) 9s ago 18m 46.7M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (12m) 29s ago 12m 301M - 4.0 e1d6a67b021e 47cbf43aaad9 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (12m) 9s ago 12m 90.3M - 4.0 e1d6a67b021e c135c54d1be1 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (9m) 29s ago 18m 17.9M - 1.3.1 1dbe0e931976 7e7218def09d 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (9m) 9s ago 18m 20.4M - 1.3.1 1dbe0e931976 8c5c0dfbeeec 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (6m) 29s ago 17m 437M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (5m) 29s ago 17m 403M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (4m) 29s ago 16m 386M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (3m) 29s ago 16m 415M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd 27fefb9c15b3 2024-04-26T15:41:16.716 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (2m) 9s ago 15m 362M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd b68352a89f2e 2024-04-26T15:41:16.717 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (2m) 9s ago 15m 329M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 61be8ee4a0ab 2024-04-26T15:41:16.717 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (103s) 9s ago 15m 323M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 78ef3c811a10 2024-04-26T15:41:16.717 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (65s) 9s ago 14m 327M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 955dfe6c53ce 2024-04-26T15:41:16.717 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (8m) 29s ago 18m 77.7M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:41:16.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:16 smithi164 ceph-mon[49952]: pgmap v325: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 6.9 KiB/s wr, 4 op/s 2024-04-26T15:41:16.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:16 smithi164 ceph-mon[49952]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:16.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:16 smithi112 ceph-mon[63158]: pgmap v325: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 6.9 KiB/s wr, 4 op/s 2024-04-26T15:41:16.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:16 smithi112 ceph-mon[63158]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:17.151 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:41:17.151 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:41:17.151 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:41:17.151 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:41:17.151 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:41:17.151 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:41:17.151 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:41:17.151 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:41:17.151 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T15:41:17.152 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:41:17.152 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:41:17.152 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:41:17.152 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:41:17.152 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:41:17.152 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 14 2024-04-26T15:41:17.152 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:41:17.152 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:41:17.521 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:41:17.522 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:41:17.522 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:41:17.522 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:41:17.522 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:41:17.522 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:41:17.522 INFO:teuthology.orchestra.run.smithi112.stdout: "osd", 2024-04-26T15:41:17.522 INFO:teuthology.orchestra.run.smithi112.stdout: "mds", 2024-04-26T15:41:17.522 INFO:teuthology.orchestra.run.smithi112.stdout: "mon", 2024-04-26T15:41:17.522 INFO:teuthology.orchestra.run.smithi112.stdout: "crash" 2024-04-26T15:41:17.522 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:41:17.522 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "16/23 daemons upgraded", 2024-04-26T15:41:17.522 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading nfs daemons", 2024-04-26T15:41:17.523 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:41:17.523 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:41:17.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:17 smithi112 ceph-mon[63158]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:17.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:17 smithi112 ceph-mon[63158]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:17.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:17 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/2542666129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:17.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:17 smithi112 ceph-mon[63158]: from='mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574]' entity='mds.foofs.smithi112.fqmway' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.112:0/2440813904"}]: dispatch 2024-04-26T15:41:17.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:17 smithi164 ceph-mon[49952]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:17.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:17 smithi164 ceph-mon[49952]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:17.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:17 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/2542666129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:17.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:17 smithi164 ceph-mon[49952]: from='mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574]' entity='mds.foofs.smithi112.fqmway' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.112:0/2440813904"}]: dispatch 2024-04-26T15:41:17.943 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:41:18.844 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:18 smithi164 ceph-mon[49952]: pgmap v326: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 6.9 KiB/s wr, 3 op/s 2024-04-26T15:41:18.844 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:18 smithi164 ceph-mon[49952]: Evicting (and blocklisting) client session 14546 (172.21.15.112:0/2440813904) 2024-04-26T15:41:18.844 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:18 smithi164 ceph-mon[49952]: from='mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574]' entity='mds.foofs.smithi112.fqmway' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.112:0/2440813904"}]': finished 2024-04-26T15:41:18.844 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:18 smithi164 ceph-mon[49952]: osdmap e153: 8 total, 8 up, 8 in 2024-04-26T15:41:18.844 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:18 smithi164 ceph-mon[49952]: from='client.34514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:18.844 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:18 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:18.844 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:18 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:18.844 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:18 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/701805821' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:41:18.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:18 smithi112 ceph-mon[63158]: pgmap v326: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 6.9 KiB/s wr, 3 op/s 2024-04-26T15:41:18.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:18 smithi112 ceph-mon[63158]: Evicting (and blocklisting) client session 14546 (172.21.15.112:0/2440813904) 2024-04-26T15:41:18.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:18 smithi112 ceph-mon[63158]: from='mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574]' entity='mds.foofs.smithi112.fqmway' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.112:0/2440813904"}]': finished 2024-04-26T15:41:18.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:18 smithi112 ceph-mon[63158]: osdmap e153: 8 total, 8 up, 8 in 2024-04-26T15:41:18.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:18 smithi112 ceph-mon[63158]: from='client.34514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:18.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:18 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:18.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:18 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:18.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:18 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/701805821' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:41:20.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: pgmap v328: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 614 B/s rd, 8.5 KiB/s wr, 4 op/s 2024-04-26T15:41:20.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: Upgrade: Updating nfs.foo.1.0.smithi164.lkwjnc (2/2) 2024-04-26T15:41:20.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:20.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: Creating key for client.nfs.foo.1.0.smithi164.lkwjnc 2024-04-26T15:41:20.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.lkwjnc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-26T15:41:20.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-26T15:41:20.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-26T15:41:20.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-26T15:41:20.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:20.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-26T15:41:20.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-26T15:41:20.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: Rados config object exists: conf-nfs.foo 2024-04-26T15:41:20.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: Creating key for client.nfs.foo.1.0.smithi164.lkwjnc-rgw 2024-04-26T15:41:20.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.lkwjnc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:41:20.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: Bind address in nfs.foo.1.0.smithi164.lkwjnc's ganesha conf is defaulting to empty 2024-04-26T15:41:20.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:20.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:20 smithi112 ceph-mon[63158]: Deploying daemon nfs.foo.1.0.smithi164.lkwjnc on smithi164 2024-04-26T15:41:20.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: pgmap v328: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 614 B/s rd, 8.5 KiB/s wr, 4 op/s 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: Upgrade: Updating nfs.foo.1.0.smithi164.lkwjnc (2/2) 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: Creating key for client.nfs.foo.1.0.smithi164.lkwjnc 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.lkwjnc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: Rados config object exists: conf-nfs.foo 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: Creating key for client.nfs.foo.1.0.smithi164.lkwjnc-rgw 2024-04-26T15:41:20.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.lkwjnc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:41:20.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: Bind address in nfs.foo.1.0.smithi164.lkwjnc's ganesha conf is defaulting to empty 2024-04-26T15:41:20.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:20.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:20 smithi164 ceph-mon[49952]: Deploying daemon nfs.foo.1.0.smithi164.lkwjnc on smithi164 2024-04-26T15:41:22.197 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:22 smithi164 ceph-mon[49952]: pgmap v329: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-26T15:41:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:22 smithi112 ceph-mon[63158]: pgmap v329: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-26T15:41:24.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:23 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:41:24.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:24.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:24.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:23 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:41:25.239 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:24 smithi112 ceph-mon[63158]: pgmap v330: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-26T15:41:25.239 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:41:25.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:24 smithi164 ceph-mon[49952]: pgmap v330: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-26T15:41:25.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:41:26.176 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:25 smithi112 ceph-mon[63158]: from='mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574]' entity='mds.foofs.smithi112.fqmway' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.164:0/187023859"}]: dispatch 2024-04-26T15:41:26.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:25 smithi164 ceph-mon[49952]: from='mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574]' entity='mds.foofs.smithi112.fqmway' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.164:0/187023859"}]: dispatch 2024-04-26T15:41:27.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:26 smithi112 ceph-mon[63158]: pgmap v331: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-04-26T15:41:27.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:26 smithi112 ceph-mon[63158]: Evicting (and blocklisting) client session 24337 (172.21.15.164:0/187023859) 2024-04-26T15:41:27.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:26 smithi112 ceph-mon[63158]: from='mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574]' entity='mds.foofs.smithi112.fqmway' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.164:0/187023859"}]': finished 2024-04-26T15:41:27.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:26 smithi112 ceph-mon[63158]: osdmap e154: 8 total, 8 up, 8 in 2024-04-26T15:41:27.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:26 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:27.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:26 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:27.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:26 smithi164 ceph-mon[49952]: pgmap v331: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-04-26T15:41:27.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:26 smithi164 ceph-mon[49952]: Evicting (and blocklisting) client session 24337 (172.21.15.164:0/187023859) 2024-04-26T15:41:27.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:26 smithi164 ceph-mon[49952]: from='mds.? [v2:172.21.15.112:6834/3277186574,v1:172.21.15.112:6835/3277186574]' entity='mds.foofs.smithi112.fqmway' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.164:0/187023859"}]': finished 2024-04-26T15:41:27.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:26 smithi164 ceph-mon[49952]: osdmap e154: 8 total, 8 up, 8 in 2024-04-26T15:41:27.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:26 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:27.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:26 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:28.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:28.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:28 smithi164 ceph-mon[49952]: pgmap v333: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-04-26T15:41:28.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:28 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:28 smithi112 ceph-mon[63158]: pgmap v333: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-04-26T15:41:28.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:28 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:29 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:29 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:29.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:29 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:29.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:29 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:30.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:30 smithi112 ceph-mon[63158]: pgmap v334: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.8 KiB/s wr, 7 op/s 2024-04-26T15:41:30.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:30 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:30.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:30 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:30.543 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:30 smithi164 ceph-mon[49952]: pgmap v334: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.8 KiB/s wr, 7 op/s 2024-04-26T15:41:30.544 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:30 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:30.544 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:30 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:31.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:31 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:31.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:31 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:32.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:31 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:32.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:31 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:32.617 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:32 smithi112 ceph-mon[63158]: pgmap v335: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-26T15:41:33.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:32 smithi164 ceph-mon[49952]: pgmap v335: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-26T15:41:34.313 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: pgmap v336: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-26T15:41:34.313 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:34.313 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:34.313 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:34.313 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:41:34.313 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:34.313 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:34.313 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:41:34.313 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.313 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.313 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.313 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: Upgrade: Setting container_image for all nfs 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi112.qswmgy"}]: dispatch 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi112.qswmgy"}]': finished 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi164.lkwjnc"}]: dispatch 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi164.lkwjnc"}]': finished 2024-04-26T15:41:34.314 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.315 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: Upgrade: Setting container_image for all nvmeof 2024-04-26T15:41:34.315 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:34 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:34.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: pgmap v336: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-26T15:41:34.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:34.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.599 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.600 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.600 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: Upgrade: Setting container_image for all nfs 2024-04-26T15:41:34.600 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:34.600 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi112.qswmgy"}]: dispatch 2024-04-26T15:41:34.600 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi112.qswmgy"}]': finished 2024-04-26T15:41:34.600 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi164.lkwjnc"}]: dispatch 2024-04-26T15:41:34.600 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi164.lkwjnc"}]': finished 2024-04-26T15:41:34.600 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:34.600 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: Upgrade: Setting container_image for all nvmeof 2024-04-26T15:41:34.600 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:34 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:35.589 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:35 smithi112 ceph-mon[63158]: Upgrade: Updating node-exporter.smithi112 (1/2) 2024-04-26T15:41:35.590 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:35 smithi112 ceph-mon[63158]: Deploying daemon node-exporter.smithi112 on smithi112 2024-04-26T15:41:35.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:35 smithi164 ceph-mon[49952]: Upgrade: Updating node-exporter.smithi112 (1/2) 2024-04-26T15:41:35.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:35 smithi164 ceph-mon[49952]: Deploying daemon node-exporter.smithi112 on smithi112 2024-04-26T15:41:36.422 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:36 smithi112 ceph-mon[63158]: pgmap v337: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 511 B/s wr, 0 op/s 2024-04-26T15:41:36.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:36 smithi164 ceph-mon[49952]: pgmap v337: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 511 B/s wr, 0 op/s 2024-04-26T15:41:38.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:38 smithi164 ceph-mon[49952]: pgmap v338: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 91 B/s rd, 459 B/s wr, 0 op/s 2024-04-26T15:41:38.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:38 smithi112 ceph-mon[63158]: pgmap v338: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 91 B/s rd, 459 B/s wr, 0 op/s 2024-04-26T15:41:39.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:41:39.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:41:40.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:40 smithi112 ceph-mon[63158]: pgmap v339: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 85 B/s rd, 426 B/s wr, 0 op/s 2024-04-26T15:41:40.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:40.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:40.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:40 smithi164 ceph-mon[49952]: pgmap v339: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 85 B/s rd, 426 B/s wr, 0 op/s 2024-04-26T15:41:40.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:40.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:42.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:42 smithi164 ceph-mon[49952]: pgmap v340: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-04-26T15:41:42.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:42 smithi164 ceph-mon[49952]: Upgrade: Updating node-exporter.smithi164 (2/2) 2024-04-26T15:41:42.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:42 smithi164 ceph-mon[49952]: Deploying daemon node-exporter.smithi164 on smithi164 2024-04-26T15:41:42.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:42 smithi112 ceph-mon[63158]: pgmap v340: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-04-26T15:41:42.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:42 smithi112 ceph-mon[63158]: Upgrade: Updating node-exporter.smithi164 (2/2) 2024-04-26T15:41:42.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:42 smithi112 ceph-mon[63158]: Deploying daemon node-exporter.smithi164 on smithi164 2024-04-26T15:41:44.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:44 smithi164 ceph-mon[49952]: pgmap v341: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:44.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:44 smithi112 ceph-mon[63158]: pgmap v341: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:46.511 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:46 smithi164 ceph-mon[49952]: pgmap v342: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:46.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:46 smithi112 ceph-mon[63158]: pgmap v342: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:48.323 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:41:48.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:48 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:48.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:48 smithi112 ceph-mon[63158]: pgmap v343: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:48.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:48 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:48.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:48 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:41:48.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:48 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:48.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:48 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:48.537 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:48 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:48.537 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:48 smithi164 ceph-mon[49952]: pgmap v343: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:48.537 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:48 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:48.537 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:48 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:41:48.537 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:48 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:48.538 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:48 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:49.008 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (10m) 20s ago 19m 25.5M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (7m) 20s ago 19m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (7m) 22s ago 19m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (10m) 20s ago 19m 54.2M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (69s) 20s ago 13m 30.9M - 19.0.0-3305-ga430f9ed 031ba2747cfd 3b013d074da0 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (45s) 22s ago 13m 15.0M - 19.0.0-3305-ga430f9ed 031ba2747cfd fb6f8c982ad9 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (10m) 20s ago 21m 528M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (9m) 22s ago 19m 469M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (8m) 20s ago 21m 62.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (8m) 22s ago 18m 47.4M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (31s) 20s ago 13m 72.0M - 5.7 031ba2747cfd c408257b905c 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (26s) 22s ago 13m 20.6M - 5.7 031ba2747cfd e390732475e6 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 starting - - - - 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 starting - - - - 2024-04-26T15:41:49.009 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (6m) 20s ago 18m 438M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:41:49.010 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (5m) 20s ago 17m 405M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:41:49.010 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (5m) 20s ago 17m 389M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:41:49.010 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (4m) 20s ago 16m 419M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd 27fefb9c15b3 2024-04-26T15:41:49.010 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (3m) 22s ago 16m 363M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd b68352a89f2e 2024-04-26T15:41:49.010 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (2m) 22s ago 16m 329M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 61be8ee4a0ab 2024-04-26T15:41:49.010 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (2m) 22s ago 15m 323M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 78ef3c811a10 2024-04-26T15:41:49.010 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (97s) 22s ago 15m 326M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 955dfe6c53ce 2024-04-26T15:41:49.010 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (9m) 20s ago 18m 77.7M - 2.33.4 514e6a882f6e 14c72aa33afd 2024-04-26T15:41:49.279 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:49 smithi112 ceph-mon[63158]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:49.279 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:49 smithi112 ceph-mon[63158]: from='client.34558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:49.279 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:49 smithi112 ceph-mon[63158]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:49.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:49 smithi164 ceph-mon[49952]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:49.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:49 smithi164 ceph-mon[49952]: from='client.34558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:49.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:49 smithi164 ceph-mon[49952]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:49.454 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:41:49.455 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:41:49.456 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 14 2024-04-26T15:41:49.456 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:41:49.456 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:41:49.814 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: "osd", 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: "nfs", 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: "mds", 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: "mon", 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: "crash" 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "18/23 daemons upgraded", 2024-04-26T15:41:49.815 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading node-exporter daemons", 2024-04-26T15:41:49.816 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:41:49.816 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:41:50.260 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:41:50.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:50 smithi164 ceph-mon[49952]: pgmap v344: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:50.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:50 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/3016963530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:50.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:50 smithi164 ceph-mon[49952]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:50.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:50 smithi112 ceph-mon[63158]: pgmap v344: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:50.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:50 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/3016963530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:50.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:50 smithi112 ceph-mon[63158]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:41:51.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:51 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:51.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:51 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/3521980387' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:41:51.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:51 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:51.623 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:51 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:51.623 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:51 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/3521980387' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:41:51.623 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:51 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:52.624 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:52 smithi112 ceph-mon[63158]: pgmap v345: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:41:52.624 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:52 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:52.624 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:52 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:52.818 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:52 smithi164 ceph-mon[49952]: pgmap v345: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:41:52.818 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:52 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:52.818 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:52 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:53.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:53.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:53 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:54.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:54.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:53 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:55.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:55 smithi112 ceph-mon[63158]: pgmap v346: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:55.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:55 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:55.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:55 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:55.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:55 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:41:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:55 smithi164 ceph-mon[49952]: pgmap v346: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:55 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:55 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:55 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:41:56.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:56 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:56.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:56 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:56.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:56 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:56.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:56 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:57.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:57 smithi164 ceph-mon[49952]: pgmap v347: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:57.365 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:57 smithi112 ceph-mon[63158]: pgmap v347: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:58.788 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: pgmap v348: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:58.788 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:58.788 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:58.788 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:58.788 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:41:58.788 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:58.788 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:58.788 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:41:58.788 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:58.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:58.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:58.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:58.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:58.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:58.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:58.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:58.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:58.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:58.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:58.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:58.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:41:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: pgmap v348: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:41:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:41:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:41:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:41:59.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:41:59.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:41:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:01.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:00 smithi164 ceph-mon[49952]: pgmap v349: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:00 smithi164 ceph-mon[49952]: Upgrade: Updating prometheus.smithi112 2024-04-26T15:42:01.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:00 smithi164 ceph-mon[49952]: Deploying daemon prometheus.smithi112 on smithi112 2024-04-26T15:42:01.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:00 smithi112 ceph-mon[63158]: pgmap v349: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:01.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:00 smithi112 ceph-mon[63158]: Upgrade: Updating prometheus.smithi112 2024-04-26T15:42:01.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:00 smithi112 ceph-mon[63158]: Deploying daemon prometheus.smithi112 on smithi112 2024-04-26T15:42:03.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:02 smithi164 ceph-mon[49952]: pgmap v350: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:42:03.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:02 smithi112 ceph-mon[63158]: pgmap v350: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:42:05.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:04 smithi164 ceph-mon[49952]: pgmap v351: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:05.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:04 smithi112 ceph-mon[63158]: pgmap v351: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:07.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:06 smithi164 ceph-mon[49952]: pgmap v352: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:07.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:06 smithi112 ceph-mon[63158]: pgmap v352: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:09.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:08 smithi164 ceph-mon[49952]: pgmap v353: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:09.183 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:08 smithi112 ceph-mon[63158]: pgmap v353: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:09.787 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:42:10.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:42:11.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:10 smithi164 ceph-mon[49952]: pgmap v354: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:11.168 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:10 smithi112 ceph-mon[63158]: pgmap v354: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:13.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:12 smithi164 ceph-mon[49952]: pgmap v355: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:42:13.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:12 smithi112 ceph-mon[63158]: pgmap v355: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:42:14.107 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:14.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:14.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:13 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:42:14.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:14.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:14.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:13 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:42:15.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:14 smithi164 ceph-mon[49952]: pgmap v356: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:15.395 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:14 smithi112 ceph-mon[63158]: pgmap v356: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:17.089 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:16 smithi112 ceph-mon[63158]: pgmap v357: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:17.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:16 smithi164 ceph-mon[49952]: pgmap v357: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:19.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:18 smithi164 ceph-mon[49952]: pgmap v358: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:19.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:18 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:19.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:18 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:19.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:18 smithi112 ceph-mon[63158]: pgmap v358: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:19.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:18 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:19.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:18 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:20.619 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:42:20.620 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:20 smithi112 ceph-mon[63158]: pgmap v359: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:20.620 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:20.620 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:20 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:20.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:20 smithi164 ceph-mon[49952]: pgmap v359: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:20.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:20 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:20.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:20 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:21.298 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:42:21.298 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (11m) 3s ago 20m 25.5M - 0.23.0 ba2b418f427c 268812a98f77 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (7m) 3s ago 20m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (7m) 31s ago 19m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (11m) 3s ago 20m 54.7M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (101s) 3s ago 14m 31.4M - 19.0.0-3305-ga430f9ed 031ba2747cfd 3b013d074da0 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (78s) 31s ago 14m 15.1M - 19.0.0-3305-ga430f9ed 031ba2747cfd fb6f8c982ad9 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (10m) 3s ago 21m 531M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (9m) 31s ago 19m 469M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (9m) 3s ago 21m 63.0M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (8m) 31s ago 19m 48.6M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (64s) 3s ago 13m 72.3M - 5.7 031ba2747cfd c408257b905c 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (58s) 31s ago 13m 69.2M - 5.7 031ba2747cfd e390732475e6 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (41s) 3s ago 20m 7944k - 1.5.0 0da6a335fe13 6abd5d4976c4 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (34s) 31s ago 19m 3376k - 1.5.0 0da6a335fe13 041edb8118f6 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (7m) 3s ago 18m 440M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:42:21.299 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (6m) 3s ago 18m 405M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:42:21.300 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (5m) 3s ago 17m 389M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:42:21.300 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (4m) 3s ago 17m 420M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd 27fefb9c15b3 2024-04-26T15:42:21.300 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (4m) 31s ago 16m 363M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd b68352a89f2e 2024-04-26T15:42:21.300 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (3m) 31s ago 16m 329M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 61be8ee4a0ab 2024-04-26T15:42:21.300 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (2m) 31s ago 16m 324M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 78ef3c811a10 2024-04-26T15:42:21.300 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (2m) 31s ago 15m 326M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 955dfe6c53ce 2024-04-26T15:42:21.300 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (8s) 3s ago 19m 45.5M - 2.43.0 a07b618ecd1d bde42e9679d6 2024-04-26T15:42:21.616 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:21 smithi112 ceph-mon[63158]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:21.617 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:21 smithi112 ceph-mon[63158]: from='client.34582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:21.729 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:42:21.729 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:42:21.729 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:42:21.729 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:42:21.729 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:42:21.729 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:42:21.730 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:42:21.730 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:42:21.730 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T15:42:21.730 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:42:21.730 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:42:21.730 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:42:21.730 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:42:21.730 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:42:21.730 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 14 2024-04-26T15:42:21.730 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:42:21.730 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:42:21.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:21 smithi164 ceph-mon[49952]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:21.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:21 smithi164 ceph-mon[49952]: from='client.34582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:22.079 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:42:22.079 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:42:22.079 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:42:22.079 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:42:22.079 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:42:22.079 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:42:22.079 INFO:teuthology.orchestra.run.smithi112.stdout: "osd", 2024-04-26T15:42:22.079 INFO:teuthology.orchestra.run.smithi112.stdout: "nfs", 2024-04-26T15:42:22.079 INFO:teuthology.orchestra.run.smithi112.stdout: "mds", 2024-04-26T15:42:22.080 INFO:teuthology.orchestra.run.smithi112.stdout: "mon", 2024-04-26T15:42:22.080 INFO:teuthology.orchestra.run.smithi112.stdout: "crash" 2024-04-26T15:42:22.080 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:42:22.080 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "18/23 daemons upgraded", 2024-04-26T15:42:22.080 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading prometheus daemons", 2024-04-26T15:42:22.080 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:42:22.080 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:42:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:22 smithi112 ceph-mon[63158]: pgmap v360: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:42:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:22 smithi112 ceph-mon[63158]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:22 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/1543473680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:22.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:22 smithi112 ceph-mon[63158]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:22.506 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:42:22.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:22 smithi164 ceph-mon[49952]: pgmap v360: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:42:22.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:22 smithi164 ceph-mon[49952]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:22.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:22 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/1543473680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:22.848 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:22 smithi164 ceph-mon[49952]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:23.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:23 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/1915119184' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:42:23.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:23 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/1915119184' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:42:24.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: pgmap v361: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:24.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:24.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:24.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:42:24.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:24.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:42:25.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: pgmap v361: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:25.099 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:42:26.913 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:26 smithi164 ceph-mon[49952]: pgmap v362: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:26.914 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:26 smithi164 ceph-mon[49952]: Upgrade: Updating alertmanager.smithi112 2024-04-26T15:42:26.914 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:26 smithi164 ceph-mon[49952]: Deploying daemon alertmanager.smithi112 on smithi112 2024-04-26T15:42:26.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:26 smithi112 ceph-mon[63158]: pgmap v362: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:26.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:26 smithi112 ceph-mon[63158]: Upgrade: Updating alertmanager.smithi112 2024-04-26T15:42:26.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:26 smithi112 ceph-mon[63158]: Deploying daemon alertmanager.smithi112 on smithi112 2024-04-26T15:42:28.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:28 smithi112 ceph-mon[63158]: pgmap v363: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:29.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:28 smithi164 ceph-mon[49952]: pgmap v363: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:30.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:30 smithi112 ceph-mon[63158]: pgmap v364: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:31.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:30 smithi164 ceph-mon[49952]: pgmap v364: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:32.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:32 smithi112 ceph-mon[63158]: pgmap v365: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:42:33.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:32 smithi164 ceph-mon[49952]: pgmap v365: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:42:34.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:34 smithi112 ceph-mon[63158]: pgmap v366: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:35.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:34 smithi164 ceph-mon[49952]: pgmap v366: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:36.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:36 smithi164 ceph-mon[49952]: pgmap v367: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:36.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:36 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:36.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:36 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:36.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:36 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:42:36.705 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:36 smithi112 ceph-mon[63158]: pgmap v367: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:36.705 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:36 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:36.706 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:36 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:36.706 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:36 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:42:38.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:38 smithi112 ceph-mon[63158]: pgmap v368: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:38.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:38 smithi164 ceph-mon[49952]: pgmap v368: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:39.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:42:39.678 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:42:40.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:40 smithi164 ceph-mon[49952]: pgmap v369: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:40.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:40.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:40.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:40 smithi112 ceph-mon[63158]: pgmap v369: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:40.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:40.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:42.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:42 smithi112 ceph-mon[63158]: pgmap v370: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:42:42.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:42 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:42.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:42 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:42.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:42 smithi164 ceph-mon[49952]: pgmap v370: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:42:42.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:42 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:42.598 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:42 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:44.596 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:44 smithi112 ceph-mon[63158]: pgmap v371: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:44.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:44 smithi164 ceph-mon[49952]: pgmap v371: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:45.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:45.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:45.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:42:45.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:42:45.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi112:9093"}]: dispatch 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi112:9093"}]: dispatch 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:45.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:45 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:46.123 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:46.123 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:42:46.123 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi112:9093"}]: dispatch 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi112:9093"}]: dispatch 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.124 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.125 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.125 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.125 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.125 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.125 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.125 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.125 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:46.125 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:45 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:47.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:46 smithi164 ceph-mon[49952]: pgmap v372: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:47.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:46 smithi112 ceph-mon[63158]: pgmap v372: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:48.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:47 smithi164 ceph-mon[49952]: Upgrade: Updating grafana.smithi112 2024-04-26T15:42:48.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:47 smithi164 ceph-mon[49952]: Deploying daemon grafana.smithi112 on smithi112 2024-04-26T15:42:48.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:47 smithi112 ceph-mon[63158]: Upgrade: Updating grafana.smithi112 2024-04-26T15:42:48.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:47 smithi112 ceph-mon[63158]: Deploying daemon grafana.smithi112 on smithi112 2024-04-26T15:42:49.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:48 smithi164 ceph-mon[49952]: pgmap v373: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:49.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:48 smithi112 ceph-mon[63158]: pgmap v373: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:51.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:50 smithi164 ceph-mon[49952]: pgmap v374: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:51.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:50 smithi112 ceph-mon[63158]: pgmap v374: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:52.934 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:42:53.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:52 smithi164 ceph-mon[49952]: pgmap v375: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:42:53.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:52 smithi112 ceph-mon[63158]: pgmap v375: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:42:53.612 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:42:53.612 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (18s) 13s ago 20m 17.0M - 0.25.0 c8568f914cd2 6c8bf7f1d176 2024-04-26T15:42:53.612 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (8m) 13s ago 20m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:42:53.612 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (8m) 63s ago 20m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:42:53.612 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (11m) 13s ago 20m 55.0M - 8.3.5 dad864ee21e9 b0875f57165e 2024-04-26T15:42:53.612 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (2m) 13s ago 14m 31.6M - 19.0.0-3305-ga430f9ed 031ba2747cfd 3b013d074da0 2024-04-26T15:42:53.612 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (110s) 63s ago 14m 15.1M - 19.0.0-3305-ga430f9ed 031ba2747cfd fb6f8c982ad9 2024-04-26T15:42:53.612 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (11m) 13s ago 22m 531M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:42:53.612 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (10m) 63s ago 20m 469M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:42:53.612 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (9m) 13s ago 22m 63.4M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:42:53.612 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (9m) 63s ago 20m 48.6M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (96s) 13s ago 14m 72.4M - 5.7 031ba2747cfd c408257b905c 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (91s) 63s ago 14m 69.2M - 5.7 031ba2747cfd e390732475e6 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (73s) 13s ago 20m 7428k - 1.5.0 0da6a335fe13 6abd5d4976c4 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (67s) 63s ago 19m 3376k - 1.5.0 0da6a335fe13 041edb8118f6 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (7m) 13s ago 19m 440M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (6m) 13s ago 18m 405M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (6m) 13s ago 18m 390M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (5m) 13s ago 17m 420M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd 27fefb9c15b3 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (4m) 63s ago 17m 363M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd b68352a89f2e 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (3m) 63s ago 17m 329M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 61be8ee4a0ab 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (3m) 63s ago 16m 324M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 78ef3c811a10 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (2m) 63s ago 16m 326M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 955dfe6c53ce 2024-04-26T15:42:53.613 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (41s) 13s ago 19m 46.7M - 2.43.0 a07b618ecd1d bde42e9679d6 2024-04-26T15:42:53.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:53 smithi112 ceph-mon[63158]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:54.035 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:42:54.035 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:42:54.035 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 14 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:42:54.036 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:42:54.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:53 smithi164 ceph-mon[49952]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:54.379 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:42:54.379 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T15:42:54.379 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": true, 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [ 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr", 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout: "osd", 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout: "nfs", 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout: "mds", 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout: "mon", 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout: "crash" 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": "18/23 daemons upgraded", 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "Currently upgrading grafana daemons", 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:42:54.380 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:42:54.792 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:42:55.060 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:54 smithi112 ceph-mon[63158]: pgmap v376: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:55.060 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:54 smithi112 ceph-mon[63158]: from='client.34606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:55.060 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:54 smithi112 ceph-mon[63158]: from='client.34610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:55.060 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:54 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/2716663174' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:55.060 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:42:55.060 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:54 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/3118416397' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:42:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:54 smithi164 ceph-mon[49952]: pgmap v376: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:54 smithi164 ceph-mon[49952]: from='client.34606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:54 smithi164 ceph-mon[49952]: from='client.34610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:54 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/2716663174' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:42:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:42:55.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:54 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/3118416397' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:42:56.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:55 smithi164 ceph-mon[49952]: from='client.34618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:56.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:55 smithi112 ceph-mon[63158]: from='client.34618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:42:57.071 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:56 smithi112 ceph-mon[63158]: pgmap v377: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:57.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:56 smithi164 ceph-mon[49952]: pgmap v377: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:59.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:58 smithi164 ceph-mon[49952]: pgmap v378: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:59.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:59.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:59.097 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:58 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:42:59.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:58 smithi112 ceph-mon[63158]: pgmap v378: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:42:59.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:59.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:42:59.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:58 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:43:00.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:59 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:00.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:42:59 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:00.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:59 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:00.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:42:59 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:00.960 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:00 smithi112 ceph-mon[63158]: pgmap v379: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:01.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:00 smithi164 ceph-mon[49952]: pgmap v379: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:03.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:02 smithi112 ceph-mon[63158]: pgmap v380: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:43:03.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:02 smithi164 ceph-mon[49952]: pgmap v380: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:43:05.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:04 smithi112 ceph-mon[63158]: pgmap v381: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:05.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:04 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:05.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:04 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:05.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:04 smithi164 ceph-mon[49952]: pgmap v381: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:05.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:04 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:05.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:04 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:06.210 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:06.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:05 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:06.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:06.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:05 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:07.211 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:06 smithi112 ceph-mon[63158]: pgmap v382: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:07.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:06 smithi164 ceph-mon[49952]: pgmap v382: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:09.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: pgmap v383: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:09.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi112:3000"}]: dispatch 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.349 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T15:43:09.351 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.353 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:08 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: pgmap v383: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:09.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi112:3000"}]: dispatch 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.462 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.463 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T15:43:09.464 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T15:43:09.465 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:43:09.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.467 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.467 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:43:09.467 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:43:09.467 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:43:09.467 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:09.467 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:08 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:10.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:09 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:43:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:09 smithi164 ceph-mon[49952]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi112:3000"}]: dispatch 2024-04-26T15:43:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:09 smithi164 ceph-mon[49952]: Upgrade: Finalizing container_image settings 2024-04-26T15:43:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:09 smithi164 ceph-mon[49952]: Upgrade: Complete! 2024-04-26T15:43:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:43:10.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:10.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:09 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:43:10.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:09 smithi112 ceph-mon[63158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi112:3000"}]: dispatch 2024-04-26T15:43:10.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:09 smithi112 ceph-mon[63158]: Upgrade: Finalizing container_image settings 2024-04-26T15:43:10.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:09 smithi112 ceph-mon[63158]: Upgrade: Complete! 2024-04-26T15:43:10.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:43:10.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:43:11.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:10 smithi164 ceph-mon[49952]: pgmap v384: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:11.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:10 smithi112 ceph-mon[63158]: pgmap v384: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:13.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:13 smithi164 ceph-mon[49952]: pgmap v385: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:43:13.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:13 smithi112 ceph-mon[63158]: pgmap v385: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:43:15.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:15 smithi164 ceph-mon[49952]: pgmap v386: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:15.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:15 smithi112 ceph-mon[63158]: pgmap v386: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:17.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:17 smithi164 ceph-mon[49952]: pgmap v387: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:17.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:17 smithi112 ceph-mon[63158]: pgmap v387: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:19.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:19 smithi164 ceph-mon[49952]: pgmap v388: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:19.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:19 smithi112 ceph-mon[63158]: pgmap v388: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:21.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:21 smithi164 ceph-mon[49952]: pgmap v389: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:21.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:21 smithi112 ceph-mon[63158]: pgmap v389: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:23.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:23 smithi164 ceph-mon[49952]: pgmap v390: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:43:23.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:23 smithi112 ceph-mon[63158]: pgmap v390: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:43:25.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:25 smithi164 ceph-mon[49952]: pgmap v391: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:25.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:25 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:43:25.458 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:25 smithi112 ceph-mon[63158]: pgmap v391: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:25.458 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:25 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:43:25.765 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T15:43:27.282 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:27 smithi112 ceph-mon[63158]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:43:27.283 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:27 smithi112 ceph-mon[63158]: pgmap v392: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:27.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:27 smithi164 ceph-mon[49952]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:43:27.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:27 smithi164 ceph-mon[49952]: pgmap v392: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:27.758 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:43:27.758 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (53s) 23s ago 21m 17.0M - 0.25.0 c8568f914cd2 6c8bf7f1d176 2024-04-26T15:43:27.758 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (9m) 23s ago 21m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:43:27.758 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (8m) 97s ago 20m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:43:27.758 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (29s) 23s ago 21m 48.0M - 9.4.12 2bacad6d85d8 a1d37c647568 2024-04-26T15:43:27.758 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (2m) 23s ago 15m 31.8M - 19.0.0-3305-ga430f9ed 031ba2747cfd 3b013d074da0 2024-04-26T15:43:27.758 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (2m) 97s ago 15m 15.1M - 19.0.0-3305-ga430f9ed 031ba2747cfd fb6f8c982ad9 2024-04-26T15:43:27.758 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (11m) 23s ago 22m 538M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:43:27.758 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (10m) 97s ago 20m 469M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:43:27.758 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (10m) 23s ago 22m 64.5M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:43:27.758 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (10m) 97s ago 20m 48.6M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:43:27.758 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (2m) 23s ago 15m 72.4M - 5.7 031ba2747cfd c408257b905c 2024-04-26T15:43:27.759 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (2m) 97s ago 14m 69.2M - 5.7 031ba2747cfd e390732475e6 2024-04-26T15:43:27.759 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (107s) 23s ago 21m 7465k - 1.5.0 0da6a335fe13 6abd5d4976c4 2024-04-26T15:43:27.759 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (101s) 97s ago 20m 3376k - 1.5.0 0da6a335fe13 041edb8118f6 2024-04-26T15:43:27.759 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (8m) 23s ago 19m 440M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:43:27.759 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (7m) 23s ago 19m 406M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:43:27.759 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (6m) 23s ago 18m 390M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:43:27.759 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (5m) 23s ago 18m 420M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd 27fefb9c15b3 2024-04-26T15:43:27.759 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (5m) 97s ago 18m 363M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd b68352a89f2e 2024-04-26T15:43:27.759 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (4m) 97s ago 17m 329M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 61be8ee4a0ab 2024-04-26T15:43:27.759 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (3m) 97s ago 17m 324M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 78ef3c811a10 2024-04-26T15:43:27.759 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (3m) 97s ago 16m 326M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 955dfe6c53ce 2024-04-26T15:43:27.759 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (75s) 23s ago 20m 49.0M - 2.43.0 a07b618ecd1d bde42e9679d6 2024-04-26T15:43:28.422 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions' 2024-04-26T15:43:29.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:29 smithi164 ceph-mon[49952]: pgmap v393: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:29.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:29 smithi164 ceph-mon[49952]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:43:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:29 smithi112 ceph-mon[63158]: pgmap v393: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:29.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:29 smithi112 ceph-mon[63158]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:43:30.415 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:43:30.415 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:43:30.415 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:43:30.415 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 14 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:43:30.416 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:43:31.036 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-26T15:43:31.337 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:31 smithi112 ceph-mon[63158]: pgmap v394: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:31.337 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:31 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/2979994725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:31.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:31 smithi164 ceph-mon[49952]: pgmap v394: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:31.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:31 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/2979994725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:43:32.798 INFO:teuthology.orchestra.run.smithi112.stdout:wait for servicemap items w/ changing names to refresh 2024-04-26T15:43:33.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:33 smithi164 ceph-mon[49952]: pgmap v395: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:43:33.393 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:33 smithi112 ceph-mon[63158]: pgmap v395: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:43:33.418 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'sleep 60' 2024-04-26T15:43:35.346 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:35 smithi112 ceph-mon[63158]: pgmap v396: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:35.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:35 smithi164 ceph-mon[49952]: pgmap v396: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:37.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:37 smithi164 ceph-mon[49952]: pgmap v397: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:37.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:37 smithi112 ceph-mon[63158]: pgmap v397: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:39.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:39 smithi164 ceph-mon[49952]: pgmap v398: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:39.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:39 smithi112 ceph-mon[63158]: pgmap v398: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:40.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:40 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:43:40.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:40 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:43:41.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:41 smithi164 ceph-mon[49952]: pgmap v399: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:41.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:41 smithi112 ceph-mon[63158]: pgmap v399: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:43.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:43 smithi164 ceph-mon[49952]: pgmap v400: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:43:43.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:43 smithi112 ceph-mon[63158]: pgmap v400: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:43:45.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:45 smithi164 ceph-mon[49952]: pgmap v401: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:45.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:45 smithi112 ceph-mon[63158]: pgmap v401: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:47.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:47 smithi164 ceph-mon[49952]: pgmap v402: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:47.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:47 smithi112 ceph-mon[63158]: pgmap v402: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:49.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:49 smithi164 ceph-mon[49952]: pgmap v403: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:49.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:49 smithi112 ceph-mon[63158]: pgmap v403: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:51.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:51 smithi164 ceph-mon[49952]: pgmap v404: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:51.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:51 smithi112 ceph-mon[63158]: pgmap v404: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:53.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:53 smithi164 ceph-mon[49952]: pgmap v405: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:43:53.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:53 smithi112 ceph-mon[63158]: pgmap v405: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:43:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:55 smithi164 ceph-mon[49952]: pgmap v406: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:55.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:55 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:43:55.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:55 smithi112 ceph-mon[63158]: pgmap v406: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:55.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:55 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:43:57.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:57 smithi164 ceph-mon[49952]: pgmap v407: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:57.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:57 smithi112 ceph-mon[63158]: pgmap v407: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:59.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:43:59 smithi164 ceph-mon[49952]: pgmap v408: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:43:59.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:43:59 smithi112 ceph-mon[63158]: pgmap v408: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:01.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:01 smithi164 ceph-mon[49952]: pgmap v409: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:01.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:01 smithi112 ceph-mon[63158]: pgmap v409: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:03.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:03 smithi164 ceph-mon[49952]: pgmap v410: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:44:03.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:03 smithi112 ceph-mon[63158]: pgmap v410: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:44:05.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:05 smithi164 ceph-mon[49952]: pgmap v411: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:05.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:05 smithi112 ceph-mon[63158]: pgmap v411: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:07.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:07 smithi164 ceph-mon[49952]: pgmap v412: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:07.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:07 smithi112 ceph-mon[63158]: pgmap v412: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:09.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:09 smithi164 ceph-mon[49952]: pgmap v413: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:09.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:44:09.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:09 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:44:09.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:09 smithi112 ceph-mon[63158]: pgmap v413: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:09.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:44:09.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:09 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:44:10.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:10 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:44:10.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:10 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:44:10.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:10 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:44:10.347 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:10 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:44:10.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:10 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:44:10.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:10 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:44:10.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:10 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:44:10.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:10 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' 2024-04-26T15:44:11.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:11 smithi112 ceph-mon[63158]: pgmap v414: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:11.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:11 smithi164 ceph-mon[49952]: pgmap v414: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:13.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:13 smithi112 ceph-mon[63158]: pgmap v415: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:44:13.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:13 smithi164 ceph-mon[49952]: pgmap v415: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:44:15.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:15 smithi112 ceph-mon[63158]: pgmap v416: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:15.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:15 smithi164 ceph-mon[49952]: pgmap v416: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:17.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:17 smithi112 ceph-mon[63158]: pgmap v417: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:17.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:17 smithi164 ceph-mon[49952]: pgmap v417: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:18.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:18 smithi112 ceph-mon[63158]: pgmap v418: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:18.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:18 smithi164 ceph-mon[49952]: pgmap v418: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:20.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:20 smithi164 ceph-mon[49952]: pgmap v419: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:20.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:20 smithi112 ceph-mon[63158]: pgmap v419: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:22.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:22 smithi164 ceph-mon[49952]: pgmap v420: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:44:22.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:22 smithi112 ceph-mon[63158]: pgmap v420: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:44:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:24 smithi164 ceph-mon[49952]: pgmap v421: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:24.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:24 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:44:24.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:24 smithi112 ceph-mon[63158]: pgmap v421: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:24.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:24 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:44:26.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:26 smithi164 ceph-mon[49952]: pgmap v422: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:26.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:26 smithi112 ceph-mon[63158]: pgmap v422: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:28.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:28 smithi164 ceph-mon[49952]: pgmap v423: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:28.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:28 smithi112 ceph-mon[63158]: pgmap v423: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:30.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:30 smithi164 ceph-mon[49952]: pgmap v424: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:30.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:30 smithi112 ceph-mon[63158]: pgmap v424: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:32.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:32 smithi164 ceph-mon[49952]: pgmap v425: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:44:32.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:32 smithi112 ceph-mon[63158]: pgmap v425: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:44:34.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:34 smithi164 ceph-mon[49952]: pgmap v426: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:34.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:34 smithi112 ceph-mon[63158]: pgmap v426: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:35.943 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T15:44:36.541 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:36 smithi112 ceph-mon[63158]: pgmap v427: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:36.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:36 smithi164 ceph-mon[49952]: pgmap v427: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:37.883 INFO:teuthology.orchestra.run.smithi112.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T15:44:37.883 INFO:teuthology.orchestra.run.smithi112.stdout:alertmanager.smithi112 smithi112 *:9093,9094 running (2m) 93s ago 22m 17.0M - 0.25.0 c8568f914cd2 6c8bf7f1d176 2024-04-26T15:44:37.883 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi112 smithi112 running (10m) 93s ago 22m 6660k - 19.0.0-3305-ga430f9ed 031ba2747cfd 151f6175ae9f 2024-04-26T15:44:37.883 INFO:teuthology.orchestra.run.smithi112.stdout:crash.smithi164 smithi164 running (10m) 2m ago 21m 6665k - 19.0.0-3305-ga430f9ed 031ba2747cfd f7c826ddf22b 2024-04-26T15:44:37.883 INFO:teuthology.orchestra.run.smithi112.stdout:grafana.smithi112 smithi112 *:3000 running (99s) 93s ago 22m 48.0M - 9.4.12 2bacad6d85d8 a1d37c647568 2024-04-26T15:44:37.883 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi112.fqmway smithi112 running (3m) 93s ago 16m 31.8M - 19.0.0-3305-ga430f9ed 031ba2747cfd 3b013d074da0 2024-04-26T15:44:37.883 INFO:teuthology.orchestra.run.smithi112.stdout:mds.foofs.smithi164.gbxewm smithi164 running (3m) 2m ago 16m 15.1M - 19.0.0-3305-ga430f9ed 031ba2747cfd fb6f8c982ad9 2024-04-26T15:44:37.883 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi112.llioug smithi112 *:8443,9283,8765 running (12m) 93s ago 23m 538M - 19.0.0-3305-ga430f9ed 031ba2747cfd b98791691b81 2024-04-26T15:44:37.883 INFO:teuthology.orchestra.run.smithi112.stdout:mgr.smithi164.tuhwus smithi164 *:8443,9283,8765 running (11m) 2m ago 21m 469M - 19.0.0-3305-ga430f9ed 031ba2747cfd b01f08cea869 2024-04-26T15:44:37.883 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi112 smithi112 running (11m) 93s ago 23m 64.5M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd d00aea383229 2024-04-26T15:44:37.883 INFO:teuthology.orchestra.run.smithi112.stdout:mon.smithi164 smithi164 running (11m) 2m ago 21m 48.6M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 2e8f0c8e5619 2024-04-26T15:44:37.883 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.0.0.smithi112.qswmgy smithi112 *:2049 running (3m) 93s ago 16m 72.4M - 5.7 031ba2747cfd c408257b905c 2024-04-26T15:44:37.884 INFO:teuthology.orchestra.run.smithi112.stdout:nfs.foo.1.0.smithi164.lkwjnc smithi164 *:2049 running (3m) 2m ago 16m 69.2M - 5.7 031ba2747cfd e390732475e6 2024-04-26T15:44:37.884 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi112 smithi112 *:9100 running (2m) 93s ago 22m 7465k - 1.5.0 0da6a335fe13 6abd5d4976c4 2024-04-26T15:44:37.884 INFO:teuthology.orchestra.run.smithi112.stdout:node-exporter.smithi164 smithi164 *:9100 running (2m) 2m ago 21m 3376k - 1.5.0 0da6a335fe13 041edb8118f6 2024-04-26T15:44:37.884 INFO:teuthology.orchestra.run.smithi112.stdout:osd.0 smithi112 running (9m) 93s ago 21m 440M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd ed1d8391102f 2024-04-26T15:44:37.884 INFO:teuthology.orchestra.run.smithi112.stdout:osd.1 smithi112 running (8m) 93s ago 20m 406M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd d2979058d0eb 2024-04-26T15:44:37.884 INFO:teuthology.orchestra.run.smithi112.stdout:osd.2 smithi112 running (7m) 93s ago 20m 390M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd a6c099476ffc 2024-04-26T15:44:37.884 INFO:teuthology.orchestra.run.smithi112.stdout:osd.3 smithi112 running (7m) 93s ago 19m 420M 1936M 19.0.0-3305-ga430f9ed 031ba2747cfd 27fefb9c15b3 2024-04-26T15:44:37.884 INFO:teuthology.orchestra.run.smithi112.stdout:osd.4 smithi164 running (6m) 2m ago 19m 363M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd b68352a89f2e 2024-04-26T15:44:37.884 INFO:teuthology.orchestra.run.smithi112.stdout:osd.5 smithi164 running (5m) 2m ago 18m 329M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 61be8ee4a0ab 2024-04-26T15:44:37.884 INFO:teuthology.orchestra.run.smithi112.stdout:osd.6 smithi164 running (5m) 2m ago 18m 324M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 78ef3c811a10 2024-04-26T15:44:37.884 INFO:teuthology.orchestra.run.smithi112.stdout:osd.7 smithi164 running (4m) 2m ago 18m 326M 2704M 19.0.0-3305-ga430f9ed 031ba2747cfd 955dfe6c53ce 2024-04-26T15:44:37.884 INFO:teuthology.orchestra.run.smithi112.stdout:prometheus.smithi112 smithi112 *:9095 running (2m) 93s ago 21m 49.0M - 2.43.0 a07b618ecd1d bde42e9679d6 2024-04-26T15:44:38.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:38 smithi112 ceph-mon[63158]: pgmap v428: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:38.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:38 smithi112 ceph-mon[63158]: from='client.34638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:44:38.563 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade status' 2024-04-26T15:44:38.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:38 smithi164 ceph-mon[49952]: pgmap v428: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:38.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:38 smithi164 ceph-mon[49952]: from='client.34638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:44:39.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:39 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:44:39.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:39 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:44:40.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:40 smithi112 ceph-mon[63158]: pgmap v429: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:40.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:40 smithi164 ceph-mon[49952]: pgmap v429: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:40.626 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:44:40.626 INFO:teuthology.orchestra.run.smithi112.stdout: "target_image": null, 2024-04-26T15:44:40.626 INFO:teuthology.orchestra.run.smithi112.stdout: "in_progress": false, 2024-04-26T15:44:40.626 INFO:teuthology.orchestra.run.smithi112.stdout: "which": "", 2024-04-26T15:44:40.626 INFO:teuthology.orchestra.run.smithi112.stdout: "services_complete": [], 2024-04-26T15:44:40.626 INFO:teuthology.orchestra.run.smithi112.stdout: "progress": null, 2024-04-26T15:44:40.626 INFO:teuthology.orchestra.run.smithi112.stdout: "message": "", 2024-04-26T15:44:40.627 INFO:teuthology.orchestra.run.smithi112.stdout: "is_paused": false 2024-04-26T15:44:40.627 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:44:41.377 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:41 smithi112 ceph-mon[63158]: from='client.34642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:44:41.402 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph health detail' 2024-04-26T15:44:41.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:41 smithi164 ceph-mon[49952]: from='client.34642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:44:42.334 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:42 smithi112 ceph-mon[63158]: pgmap v430: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:44:42.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:42 smithi164 ceph-mon[49952]: pgmap v430: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:44:43.495 INFO:teuthology.orchestra.run.smithi112.stdout:HEALTH_OK 2024-04-26T15:44:44.158 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions' 2024-04-26T15:44:44.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:44 smithi112 ceph-mon[63158]: pgmap v431: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:44.443 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:44 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/3535194350' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:44:44.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:44 smithi164 ceph-mon[49952]: pgmap v431: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:44.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:44 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/3535194350' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T15:44:46.170 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-26T15:44:46.170 INFO:teuthology.orchestra.run.smithi112.stdout: "mon": { 2024-04-26T15:44:46.170 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:44:46.170 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:44:46.170 INFO:teuthology.orchestra.run.smithi112.stdout: "mgr": { 2024-04-26T15:44:46.170 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:44:46.170 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:44:46.170 INFO:teuthology.orchestra.run.smithi112.stdout: "osd": { 2024-04-26T15:44:46.171 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T15:44:46.171 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:44:46.171 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-04-26T15:44:46.171 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T15:44:46.171 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:44:46.171 INFO:teuthology.orchestra.run.smithi112.stdout: "overall": { 2024-04-26T15:44:46.171 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 14 2024-04-26T15:44:46.171 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:44:46.171 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-26T15:44:46.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:46 smithi112 ceph-mon[63158]: pgmap v432: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:46.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:46 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/2345347492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:44:46.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:46 smithi164 ceph-mon[49952]: pgmap v432: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:46.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:46 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/2345347492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:44:46.916 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-26T15:44:48.408 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:48 smithi112 ceph-mon[63158]: pgmap v433: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:48.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:48 smithi164 ceph-mon[49952]: pgmap v433: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:48.905 INFO:teuthology.orchestra.run.smithi112.stdout:true 2024-04-26T15:44:49.559 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:49 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/684243597' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:44:49.584 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-26T15:44:49.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:49 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/684243597' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:44:50.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:50 smithi112 ceph-mon[63158]: pgmap v434: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:50.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:50 smithi164 ceph-mon[49952]: pgmap v434: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:51.513 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)" 2024-04-26T15:44:52.303 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-04-26T15:44:52.304 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-04-26T15:44:52.304 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-26T15:44:52.606 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:52 smithi112 ceph-mon[63158]: pgmap v435: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:44:52.606 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:52 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/4013820246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:44:52.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:52 smithi164 ceph-mon[49952]: pgmap v435: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:44:52.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:52 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/4013820246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T15:44:54.294 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-26T15:44:54.294 INFO:teuthology.orchestra.run.smithi112.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-26T15:21:25.497200Z", "last_refresh": "2024-04-26T15:43:03.931308Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-26T15:21:18.687347Z", "last_refresh": "2024-04-26T15:41:50.237306Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-04-26T15:21:22.362098Z", "last_refresh": "2024-04-26T15:43:03.931453Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-26T15:27:59.824257Z", "last_refresh": "2024-04-26T15:41:50.237927Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-26T15:21:17.390738Z", "last_refresh": "2024-04-26T15:41:50.237402Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi112:172.21.15.112=smithi112", "smithi164:172.21.15.164=smithi164"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-26T15:22:10.216652Z", "last_refresh": "2024-04-26T15:41:50.237480Z", "running": 2, "size": 2}}, {"events": ["2024-04-26T15:44:09.502749Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-04-26T15:28:20.062226Z", "last_refresh": "2024-04-26T15:41:50.238016Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-26T15:21:23.960767Z", "last_refresh": "2024-04-26T15:41:50.237556Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", "last_refresh": "2024-04-26T15:41:50.237631Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-26T15:21:20.469679Z", "last_refresh": "2024-04-26T15:43:03.931595Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-26T15:44:54.603 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:54 smithi112 ceph-mon[63158]: pgmap v436: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:54.603 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:54 smithi112 ceph-mon[63158]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:44:54.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:54 smithi164 ceph-mon[49952]: pgmap v436: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:54.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:54 smithi164 ceph-mon[49952]: from='mgr.34102 172.21.15.112:0/1005648946' entity='mgr.smithi112.llioug' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T15:44:54.892 INFO:tasks.cephadm:nfs.foo has 2/2 2024-04-26T15:44:54.892 DEBUG:teuthology.parallel:result is None 2024-04-26T15:44:54.892 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-26T15:44:54.903 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:44:54.903 DEBUG:teuthology.orchestra.run.smithi112:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-04-26T15:44:54.927 INFO:teuthology.orchestra.run.smithi112.stderr:+ umount /mnt/foo 2024-04-26T15:44:54.938 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T15:44:54.955 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:44:54.955 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-04-26T15:44:55.615 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:55 smithi112 ceph-mon[63158]: from='client.34662 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:44:55.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:55 smithi164 ceph-mon[49952]: from='client.34662 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:44:56.557 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:56 smithi112 ceph-mon[63158]: pgmap v437: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:56.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:56 smithi164 ceph-mon[49952]: pgmap v437: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:57.039 INFO:teuthology.orchestra.run.smithi112.stdout: "foo" 2024-04-26T15:44:57.592 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:57 smithi112 ceph-mon[63158]: from='client.34666 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:44:57.614 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-04-26T15:44:57.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:57 smithi164 ceph-mon[49952]: from='client.34666 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:44:58.582 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:44:58 smithi112 ceph-mon[63158]: pgmap v438: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:58.748 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:44:58 smithi164 ceph-mon[49952]: pgmap v438: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:44:59.694 INFO:teuthology.orchestra.run.smithi112.stdout:[ 2024-04-26T15:44:59.694 INFO:teuthology.orchestra.run.smithi112.stdout: { 2024-04-26T15:44:59.694 INFO:teuthology.orchestra.run.smithi112.stdout: "access_type": "RW", 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "clients": [], 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "cluster_id": "foo", 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "export_id": 1, 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "fsal": { 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "fs_name": "foofs", 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "name": "CEPH", 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "user_id": "nfs.foo.1" 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "path": "/", 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "protocols": [ 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: 4 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: ], 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "pseudo": "/fake", 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "security_label": true, 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "squash": "none", 2024-04-26T15:44:59.695 INFO:teuthology.orchestra.run.smithi112.stdout: "transports": [ 2024-04-26T15:44:59.696 INFO:teuthology.orchestra.run.smithi112.stdout: "TCP" 2024-04-26T15:44:59.696 INFO:teuthology.orchestra.run.smithi112.stdout: ] 2024-04-26T15:44:59.696 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-04-26T15:44:59.696 INFO:teuthology.orchestra.run.smithi112.stdout:] 2024-04-26T15:45:00.340 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-04-26T15:45:00.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:45:00 smithi112 ceph-mon[63158]: pgmap v439: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:45:00.642 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:45:00 smithi112 ceph-mon[63158]: from='client.34670 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:45:00.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:45:00 smithi164 ceph-mon[49952]: pgmap v439: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:45:00.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:45:00 smithi164 ceph-mon[49952]: from='client.34670 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:45:02.162 INFO:teuthology.orchestra.run.smithi112.stdout:foo rec-0000000000000004:nfs.foo.1 2024-04-26T15:45:02.162 INFO:teuthology.orchestra.run.smithi112.stdout:foo grace 2024-04-26T15:45:02.162 INFO:teuthology.orchestra.run.smithi112.stdout:foo rec-0000000000000004:nfs.foo.0 2024-04-26T15:45:02.162 INFO:teuthology.orchestra.run.smithi112.stdout:foo conf-nfs.foo 2024-04-26T15:45:02.162 INFO:teuthology.orchestra.run.smithi112.stdout:foo export-1 2024-04-26T15:45:02.707 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:45:02 smithi112 ceph-mon[63158]: pgmap v440: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:45:02.707 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:45:02 smithi112 ceph-mon[63158]: mgrmap e42: smithi112.llioug(active, since 11m), standbys: smithi164.tuhwus 2024-04-26T15:45:02.731 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T15:45:02.744 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:45:02.744 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 -- bash -c 'set -ex 2024-04-26T15:45:02.744 DEBUG:teuthology.orchestra.run.smithi112:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-04-26T15:45:02.744 DEBUG:teuthology.orchestra.run.smithi112:> ' 2024-04-26T15:45:02.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:45:02 smithi164 ceph-mon[49952]: pgmap v440: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-26T15:45:02.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:45:02 smithi164 ceph-mon[49952]: mgrmap e42: smithi112.llioug(active, since 11m), standbys: smithi164.tuhwus 2024-04-26T15:45:04.526 INFO:teuthology.orchestra.run.smithi112.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-04-26T15:45:04.710 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:45:04 smithi112 ceph-mon[63158]: pgmap v441: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:45:04.755 INFO:teuthology.orchestra.run.smithi112.stderr:+ [[ 6 -gt 2 ]] 2024-04-26T15:45:04.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:45:04 smithi164 ceph-mon[49952]: pgmap v441: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T15:45:05.408 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-26T15:45:05.417 INFO:tasks.cephadm:Teardown begin 2024-04-26T15:45:05.417 DEBUG:teuthology.orchestra.run.smithi112:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T15:45:05.444 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T15:45:05.476 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-26T15:45:05.476 DEBUG:teuthology.orchestra.run.smithi112:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T15:45:05.499 DEBUG:teuthology.orchestra.run.smithi164:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T15:45:05.533 INFO:tasks.cephadm:Stopping all daemons... 2024-04-26T15:45:05.533 INFO:tasks.cephadm.mon.smithi112:Stopping mon.smithi112... 2024-04-26T15:45:05.533 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi112 2024-04-26T15:45:05.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:45:05 smithi112 ceph-mon[63158]: from='client.? 172.21.15.112:0/3204826030' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-26T15:45:05.711 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:45:05 smithi112 systemd[1]: Stopping Ceph mon.smithi112 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:45:05.847 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:45:05 smithi164 ceph-mon[49952]: from='client.? 172.21.15.112:0/3204826030' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-26T15:45:06.143 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:45:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112[63131]: 2024-04-26T15:45:05.818+0000 7f04713e4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi112 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:45:06.143 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:45:05 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112[63131]: 2024-04-26T15:45:05.818+0000 7f04713e4700 -1 mon.smithi112@0(leader) e3 *** Got Signal Terminated *** 2024-04-26T15:45:06.144 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:45:05 smithi112 podman[109120]: 2024-04-26 15:45:05.849676434 +0000 UTC m=+0.125101300 container died d00aea38322941450000dd692173de636714dd99a46a29eb2783feab7cb516c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:45:06.460 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:45:06 smithi112 podman[109120]: 2024-04-26 15:45:06.252396837 +0000 UTC m=+0.527821703 container cleanup d00aea38322941450000dd692173de636714dd99a46a29eb2783feab7cb516c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-26T15:45:06.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 26 15:45:06 smithi112 bash[109120]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi112 2024-04-26T15:45:06.595 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi112.service' 2024-04-26T15:45:06.648 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T15:45:06.648 INFO:tasks.cephadm.mon.smithi112:Stopped mon.smithi112 2024-04-26T15:45:06.648 INFO:tasks.cephadm.mon.smithi164:Stopping mon.smithi164... 2024-04-26T15:45:06.648 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi164 2024-04-26T15:45:06.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:45:06 smithi164 systemd[1]: Stopping Ceph mon.smithi164 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:45:07.339 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:45:06 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164[49948]: 2024-04-26T15:45:06.963+0000 7fb2134ec700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi164 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:45:07.339 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:45:06 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164[49948]: 2024-04-26T15:45:06.963+0000 7fb2134ec700 -1 mon.smithi164@1(peon) e3 *** Got Signal Terminated *** 2024-04-26T15:45:07.339 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:45:07 smithi164 podman[75063]: 2024-04-26 15:45:07.025075784 +0000 UTC m=+0.180323633 container died 2e8f0c8e56199e0c8a86883d9c22d1f74daca03bf20cf4c0b6639522c14fb07e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=) 2024-04-26T15:45:07.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:45:07 smithi164 podman[75063]: 2024-04-26 15:45:07.513964936 +0000 UTC m=+0.669212789 container cleanup 2e8f0c8e56199e0c8a86883d9c22d1f74daca03bf20cf4c0b6639522c14fb07e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-26T15:45:07.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Apr 26 15:45:07 smithi164 bash[75063]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-mon-smithi164 2024-04-26T15:45:07.889 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@mon.smithi164.service' 2024-04-26T15:45:07.936 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T15:45:07.936 INFO:tasks.cephadm.mon.smithi164:Stopped mon.smithi164 2024-04-26T15:45:07.936 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-26T15:45:07.936 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.0 2024-04-26T15:45:08.210 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:07 smithi112 systemd[1]: Stopping Ceph osd.0 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:45:08.711 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:08 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0[69161]: 2024-04-26T15:45:08.231+0000 7f5236926700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:45:08.711 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:08 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0[69161]: 2024-04-26T15:45:08.231+0000 7f5236926700 -1 osd.0 154 *** Got signal Terminated *** 2024-04-26T15:45:08.711 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:08 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0[69161]: 2024-04-26T15:45:08.231+0000 7f5236926700 -1 osd.0 154 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:45:13.961 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:13 smithi112 podman[109253]: 2024-04-26 15:45:13.656875986 +0000 UTC m=+5.527909743 container died ed1d8391102ff5ad9df5a1ee94bd79bde3da2872abf55d99895d97574e39ffec (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:45:14.324 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:14 smithi112 podman[109253]: 2024-04-26 15:45:14.073653697 +0000 UTC m=+5.944687458 container cleanup ed1d8391102ff5ad9df5a1ee94bd79bde3da2872abf55d99895d97574e39ffec (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, ceph=True) 2024-04-26T15:45:14.324 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:14 smithi112 bash[109253]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0 2024-04-26T15:45:14.710 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:14 smithi112 podman[109270]: 2024-04-26 15:45:14.325285336 +0000 UTC m=+0.663474799 container remove ed1d8391102ff5ad9df5a1ee94bd79bde3da2872abf55d99895d97574e39ffec (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T15:45:15.094 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:14 smithi112 podman[109363]: 2024-04-26 15:45:14.653462021 +0000 UTC m=+0.017670568 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:45:15.095 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:14 smithi112 podman[109363]: 2024-04-26 15:45:14.776469678 +0000 UTC m=+0.140678204 container create 9c05ca65c4f4cdf86335fb06a723ffe01d7db194cfd63ad64c7e3d9c3d56110b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T15:45:15.095 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:14 smithi112 podman[109363]: 2024-04-26 15:45:14.995877988 +0000 UTC m=+0.360086510 container init 9c05ca65c4f4cdf86335fb06a723ffe01d7db194cfd63ad64c7e3d9c3d56110b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T15:45:15.095 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:14 smithi112 podman[109363]: 2024-04-26 15:45:14.999355359 +0000 UTC m=+0.363563885 container start 9c05ca65c4f4cdf86335fb06a723ffe01d7db194cfd63ad64c7e3d9c3d56110b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T15:45:15.095 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:15 smithi112 podman[109363]: 2024-04-26 15:45:15.095997197 +0000 UTC m=+0.460205725 container attach 9c05ca65c4f4cdf86335fb06a723ffe01d7db194cfd63ad64c7e3d9c3d56110b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-26T15:45:15.460 INFO:journalctl@ceph.osd.0.smithi112.stdout:Apr 26 15:45:15 smithi112 podman[109363]: 2024-04-26 15:45:15.302927315 +0000 UTC m=+0.667135852 container died 9c05ca65c4f4cdf86335fb06a723ffe01d7db194cfd63ad64c7e3d9c3d56110b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-26T15:45:15.891 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.0.service' 2024-04-26T15:45:15.928 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T15:45:15.928 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-26T15:45:15.928 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-26T15:45:15.928 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.1 2024-04-26T15:45:16.211 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:16 smithi112 systemd[1]: Stopping Ceph osd.1 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:45:16.710 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:16 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[71874]: 2024-04-26T15:45:16.294+0000 7f13b16a8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:45:16.711 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:16 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[71874]: 2024-04-26T15:45:16.294+0000 7f13b16a8700 -1 osd.1 154 *** Got signal Terminated *** 2024-04-26T15:45:16.711 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:16 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1[71874]: 2024-04-26T15:45:16.294+0000 7f13b16a8700 -1 osd.1 154 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:45:21.961 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:21 smithi112 podman[109471]: 2024-04-26 15:45:21.69225278 +0000 UTC m=+5.537570699 container died d2979058d0eb20ffc25625b25a6441cd00843f7ad4679c40c0c1faad60a23526 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T15:45:22.328 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:22 smithi112 podman[109471]: 2024-04-26 15:45:22.069870419 +0000 UTC m=+5.915188343 container cleanup d2979058d0eb20ffc25625b25a6441cd00843f7ad4679c40c0c1faad60a23526 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:45:22.328 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:22 smithi112 bash[109471]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1 2024-04-26T15:45:22.679 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:22 smithi112 podman[109488]: 2024-04-26 15:45:22.329441067 +0000 UTC m=+0.634593027 container remove d2979058d0eb20ffc25625b25a6441cd00843f7ad4679c40c0c1faad60a23526 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T15:45:22.960 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:22 smithi112 podman[109581]: 2024-04-26 15:45:22.680409427 +0000 UTC m=+0.105681298 container create ea0621ca7aca6cb91770d942b0a27aec46c774abdfb26bb1a5705569451c3a10 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5) 2024-04-26T15:45:22.961 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:22 smithi112 podman[109581]: 2024-04-26 15:45:22.593464584 +0000 UTC m=+0.018736454 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:45:22.961 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:22 smithi112 podman[109581]: 2024-04-26 15:45:22.908131375 +0000 UTC m=+0.333403246 container init ea0621ca7aca6cb91770d942b0a27aec46c774abdfb26bb1a5705569451c3a10 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T15:45:22.961 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:22 smithi112 podman[109581]: 2024-04-26 15:45:22.911511815 +0000 UTC m=+0.336783688 container start ea0621ca7aca6cb91770d942b0a27aec46c774abdfb26bb1a5705569451c3a10 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True) 2024-04-26T15:45:23.460 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:23 smithi112 podman[109581]: 2024-04-26 15:45:23.008204064 +0000 UTC m=+0.433475945 container attach ea0621ca7aca6cb91770d942b0a27aec46c774abdfb26bb1a5705569451c3a10 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T15:45:23.461 INFO:journalctl@ceph.osd.1.smithi112.stdout:Apr 26 15:45:23 smithi112 podman[109581]: 2024-04-26 15:45:23.20469207 +0000 UTC m=+0.629963943 container died ea0621ca7aca6cb91770d942b0a27aec46c774abdfb26bb1a5705569451c3a10 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-26T15:45:23.889 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.1.service' 2024-04-26T15:45:23.926 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T15:45:23.926 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-26T15:45:23.926 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-26T15:45:23.926 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.2 2024-04-26T15:45:24.211 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:24 smithi112 systemd[1]: Stopping Ceph osd.2 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:45:24.711 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:24 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2[74515]: 2024-04-26T15:45:24.283+0000 7f90d32d3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:45:24.711 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:24 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2[74515]: 2024-04-26T15:45:24.283+0000 7f90d32d3700 -1 osd.2 154 *** Got signal Terminated *** 2024-04-26T15:45:24.711 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:24 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2[74515]: 2024-04-26T15:45:24.283+0000 7f90d32d3700 -1 osd.2 154 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:45:29.961 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:29 smithi112 podman[109718]: 2024-04-26 15:45:29.670244038 +0000 UTC m=+5.520878283 container died a6c099476ffcc9c846927963e9943af17d937a9375d0469e166cb3c340a5b9a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T15:45:30.461 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:30 smithi112 podman[109718]: 2024-04-26 15:45:30.100195188 +0000 UTC m=+5.950829431 container cleanup a6c099476ffcc9c846927963e9943af17d937a9375d0469e166cb3c340a5b9a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-26T15:45:30.461 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:30 smithi112 bash[109718]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2 2024-04-26T15:45:30.461 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:30 smithi112 podman[109737]: 2024-04-26 15:45:30.335187083 +0000 UTC m=+0.668489290 container remove a6c099476ffcc9c846927963e9943af17d937a9375d0469e166cb3c340a5b9a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:45:31.021 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:30 smithi112 podman[109829]: 2024-04-26 15:45:30.664839076 +0000 UTC m=+0.018969058 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:45:31.021 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:30 smithi112 podman[109829]: 2024-04-26 15:45:30.77812486 +0000 UTC m=+0.132254890 container create 6a08d0056604d1124b09dd0c9808082a536a924c8748b4bd34fcc39b3d524a13 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T15:45:31.021 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:31 smithi112 podman[109829]: 2024-04-26 15:45:31.022463986 +0000 UTC m=+0.376593958 container init 6a08d0056604d1124b09dd0c9808082a536a924c8748b4bd34fcc39b3d524a13 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T15:45:31.330 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:31 smithi112 podman[109829]: 2024-04-26 15:45:31.025766846 +0000 UTC m=+0.379896822 container start 6a08d0056604d1124b09dd0c9808082a536a924c8748b4bd34fcc39b3d524a13 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-26T15:45:31.330 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:31 smithi112 podman[109829]: 2024-04-26 15:45:31.09757828 +0000 UTC m=+0.451708259 container attach 6a08d0056604d1124b09dd0c9808082a536a924c8748b4bd34fcc39b3d524a13 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-26T15:45:31.710 INFO:journalctl@ceph.osd.2.smithi112.stdout:Apr 26 15:45:31 smithi112 podman[109829]: 2024-04-26 15:45:31.331699802 +0000 UTC m=+0.685829775 container died 6a08d0056604d1124b09dd0c9808082a536a924c8748b4bd34fcc39b3d524a13 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-04-26T15:45:31.890 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.2.service' 2024-04-26T15:45:31.926 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T15:45:31.926 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-26T15:45:31.926 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-26T15:45:31.927 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.3 2024-04-26T15:45:32.210 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:32 smithi112 systemd[1]: Stopping Ceph osd.3 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:45:32.711 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:32 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3[77244]: 2024-04-26T15:45:32.261+0000 7f0d01574700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:45:32.711 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:32 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3[77244]: 2024-04-26T15:45:32.261+0000 7f0d01574700 -1 osd.3 154 *** Got signal Terminated *** 2024-04-26T15:45:32.711 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:32 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3[77244]: 2024-04-26T15:45:32.261+0000 7f0d01574700 -1 osd.3 154 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:45:34.470 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:34 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:34.115+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:35.366 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:35 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:35.361+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.096725+0000 front 2024-04-26T15:45:11.096625+0000 (oldest deadline 2024-04-26T15:45:35.196510+0000) 2024-04-26T15:45:35.366 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:35 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:35.112+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:35.367 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:35 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:35.357+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:35.367 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:35 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:35.118+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:36.329 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:36 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:36.071+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:36.330 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:36 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:36.112+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:36.461 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:36 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3[77244]: 2024-04-26T15:45:36.089+0000 7f0cf9b60700 -1 osd.3 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.305792+0000 front 2024-04-26T15:45:09.305913+0000 (oldest deadline 2024-04-26T15:45:35.205866+0000) 2024-04-26T15:45:36.597 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:36 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:36.357+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.096725+0000 front 2024-04-26T15:45:11.096625+0000 (oldest deadline 2024-04-26T15:45:35.196510+0000) 2024-04-26T15:45:36.597 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:36 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:36.330+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:37.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:37 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:37.086+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:37.347 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:37 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:37.064+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:37.460 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:37 smithi112 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3[77244]: 2024-04-26T15:45:37.046+0000 7f0cf9b60700 -1 osd.3 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.305792+0000 front 2024-04-26T15:45:09.305913+0000 (oldest deadline 2024-04-26T15:45:35.205866+0000) 2024-04-26T15:45:37.847 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:37 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:37.388+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.096725+0000 front 2024-04-26T15:45:11.096625+0000 (oldest deadline 2024-04-26T15:45:35.196510+0000) 2024-04-26T15:45:37.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:37 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:37.360+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:38.154 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:37 smithi112 podman[109934]: 2024-04-26 15:45:37.89261215 +0000 UTC m=+5.741194241 container died 27fefb9c15b304ec42d0dc5082025426dd4bd93ea0dd19a7521d1faf1ac35e1a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-26T15:45:38.347 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:38 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:38.026+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:38.348 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:38 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:38.125+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:38.461 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:38 smithi112 podman[109934]: 2024-04-26 15:45:38.311882747 +0000 UTC m=+6.160464831 container cleanup 27fefb9c15b304ec42d0dc5082025426dd4bd93ea0dd19a7521d1faf1ac35e1a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T15:45:38.461 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:38 smithi112 bash[109934]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3 2024-04-26T15:45:38.847 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:38 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:38.395+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.096725+0000 front 2024-04-26T15:45:11.096625+0000 (oldest deadline 2024-04-26T15:45:35.196510+0000) 2024-04-26T15:45:38.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:38 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:38.408+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:38.929 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:38 smithi112 podman[109951]: 2024-04-26 15:45:38.521909964 +0000 UTC m=+0.631484678 container remove 27fefb9c15b304ec42d0dc5082025426dd4bd93ea0dd19a7521d1faf1ac35e1a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-26T15:45:39.211 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:38 smithi112 podman[110044]: 2024-04-26 15:45:38.930961371 +0000 UTC m=+0.112873606 container create 7faaaa5d311e2c3e81dbd8732e5ec8b00a08b8a24d870c4c71d146ee486283ac (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-26T15:45:39.211 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:38 smithi112 podman[110044]: 2024-04-26 15:45:38.836524496 +0000 UTC m=+0.018436701 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:45:39.211 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:39 smithi112 podman[110044]: 2024-04-26 15:45:39.166765393 +0000 UTC m=+0.348677587 container init 7faaaa5d311e2c3e81dbd8732e5ec8b00a08b8a24d870c4c71d146ee486283ac (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-26T15:45:39.211 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:39 smithi112 podman[110044]: 2024-04-26 15:45:39.170103525 +0000 UTC m=+0.352015719 container start 7faaaa5d311e2c3e81dbd8732e5ec8b00a08b8a24d870c4c71d146ee486283ac (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-26T15:45:39.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:39 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:39.118+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:39.347 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:39 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:39.074+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:39.711 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:39 smithi112 podman[110044]: 2024-04-26 15:45:39.275235705 +0000 UTC m=+0.457147918 container attach 7faaaa5d311e2c3e81dbd8732e5ec8b00a08b8a24d870c4c71d146ee486283ac (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T15:45:39.711 INFO:journalctl@ceph.osd.3.smithi112.stdout:Apr 26 15:45:39 smithi112 podman[110044]: 2024-04-26 15:45:39.463360749 +0000 UTC m=+0.645272942 container died 7faaaa5d311e2c3e81dbd8732e5ec8b00a08b8a24d870c4c71d146ee486283ac (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=) 2024-04-26T15:45:39.847 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:39 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:39.358+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.096725+0000 front 2024-04-26T15:45:11.096625+0000 (oldest deadline 2024-04-26T15:45:35.196510+0000) 2024-04-26T15:45:39.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:39 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:39.372+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:40.059 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.3.service' 2024-04-26T15:45:40.095 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T15:45:40.096 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-26T15:45:40.096 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-26T15:45:40.096 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.4 2024-04-26T15:45:40.347 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:40 smithi164 systemd[1]: Stopping Ceph osd.4 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:45:40.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:40 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:40.070+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:40.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:40 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:40.075+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:40.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:40 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:40.361+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:40.847 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:40 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:40.398+0000 7f90fe8bc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:45:40.847 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:40 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:40.398+0000 7f90fe8bc700 -1 osd.4 154 *** Got signal Terminated *** 2024-04-26T15:45:40.847 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:40 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:40.398+0000 7f90fe8bc700 -1 osd.4 154 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:45:40.847 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:40 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:40.407+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.096725+0000 front 2024-04-26T15:45:11.096625+0000 (oldest deadline 2024-04-26T15:45:35.196510+0000) 2024-04-26T15:45:41.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:41 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:41.084+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:41.347 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:41 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:41.056+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:41.847 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:41 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:41.389+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.096725+0000 front 2024-04-26T15:45:11.096625+0000 (oldest deadline 2024-04-26T15:45:35.196510+0000) 2024-04-26T15:45:41.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:41 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:41.361+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:42.347 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:42 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:42.105+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:42.348 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:42 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:42.083+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:42.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:42 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:42.354+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:42.847 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:42 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:42.376+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.096725+0000 front 2024-04-26T15:45:11.096625+0000 (oldest deadline 2024-04-26T15:45:35.196510+0000) 2024-04-26T15:45:42.848 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:42 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:42.376+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:16.897845+0000 front 2024-04-26T15:45:16.897875+0000 (oldest deadline 2024-04-26T15:45:42.197755+0000) 2024-04-26T15:45:43.329 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:43 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:43.069+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:43.329 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:43 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:43.061+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:43.329 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:43 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:43.061+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.420292+0000 front 2024-04-26T15:45:19.420331+0000 (oldest deadline 2024-04-26T15:45:42.320017+0000) 2024-04-26T15:45:43.597 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:43 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:43.365+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:43.597 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:43 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:43.329+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.096725+0000 front 2024-04-26T15:45:11.096625+0000 (oldest deadline 2024-04-26T15:45:35.196510+0000) 2024-04-26T15:45:43.598 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:43 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:43.329+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:16.897845+0000 front 2024-04-26T15:45:16.897875+0000 (oldest deadline 2024-04-26T15:45:42.197755+0000) 2024-04-26T15:45:44.347 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:44 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:44.118+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:44.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:44 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:44.052+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:44.348 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:44 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:44.052+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.420292+0000 front 2024-04-26T15:45:19.420331+0000 (oldest deadline 2024-04-26T15:45:42.320017+0000) 2024-04-26T15:45:44.847 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:44 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:44.369+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.096725+0000 front 2024-04-26T15:45:11.096625+0000 (oldest deadline 2024-04-26T15:45:35.196510+0000) 2024-04-26T15:45:44.848 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:44 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:44.369+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:16.897845+0000 front 2024-04-26T15:45:16.897875+0000 (oldest deadline 2024-04-26T15:45:42.197755+0000) 2024-04-26T15:45:44.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:44 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:44.411+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:44.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:44 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:44.411+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:45.348 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:45 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:45.094+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:45.348 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:45 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:45.094+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.420292+0000 front 2024-04-26T15:45:19.420331+0000 (oldest deadline 2024-04-26T15:45:42.320017+0000) 2024-04-26T15:45:45.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:45 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:45.077+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:45.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:45 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:45.077+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:45.764 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:45 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:45.411+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:45.764 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:45 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:45.411+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:45.764 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:45 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:45.373+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.096725+0000 front 2024-04-26T15:45:11.096625+0000 (oldest deadline 2024-04-26T15:45:35.196510+0000) 2024-04-26T15:45:45.764 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:45 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4[54906]: 2024-04-26T15:45:45.373+0000 7f90faeb0700 -1 osd.4 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:16.897845+0000 front 2024-04-26T15:45:16.897875+0000 (oldest deadline 2024-04-26T15:45:42.197755+0000) 2024-04-26T15:45:46.041 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:45 smithi164 podman[75228]: 2024-04-26 15:45:45.76515669 +0000 UTC m=+5.477071022 container died b68352a89f2e5137c025a2e2a6556c51a993656435747fb9a79467ccf76f2c57 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T15:45:46.347 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:46 smithi164 podman[75228]: 2024-04-26 15:45:46.249915338 +0000 UTC m=+5.961829711 container cleanup b68352a89f2e5137c025a2e2a6556c51a993656435747fb9a79467ccf76f2c57 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-26T15:45:46.348 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:46 smithi164 bash[75228]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4 2024-04-26T15:45:46.348 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:46 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:46.068+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:46.348 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:46 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:46.068+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.420292+0000 front 2024-04-26T15:45:19.420331+0000 (oldest deadline 2024-04-26T15:45:42.320017+0000) 2024-04-26T15:45:46.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:46 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:46.083+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:46.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:46 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:46.083+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:46.847 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:46 smithi164 podman[75277]: 2024-04-26 15:45:46.50971835 +0000 UTC m=+0.741985437 container remove b68352a89f2e5137c025a2e2a6556c51a993656435747fb9a79467ccf76f2c57 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-26T15:45:46.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:46 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:46.409+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:46.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:46 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:46.409+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:47.195 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:46 smithi164 podman[75372]: 2024-04-26 15:45:46.86922308 +0000 UTC m=+0.112507164 container create 63cfff8a7a451f56dc83732947be7b2b29540adc2c9a858bfa2023399a90e0bc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-26T15:45:47.195 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:46 smithi164 podman[75372]: 2024-04-26 15:45:46.775017738 +0000 UTC m=+0.018301827 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:45:47.195 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:47 smithi164 podman[75372]: 2024-04-26 15:45:47.097134806 +0000 UTC m=+0.340418895 container init 63cfff8a7a451f56dc83732947be7b2b29540adc2c9a858bfa2023399a90e0bc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-26T15:45:47.196 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:47 smithi164 podman[75372]: 2024-04-26 15:45:47.100413645 +0000 UTC m=+0.343697727 container start 63cfff8a7a451f56dc83732947be7b2b29540adc2c9a858bfa2023399a90e0bc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5) 2024-04-26T15:45:47.196 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:47 smithi164 podman[75372]: 2024-04-26 15:45:47.196742448 +0000 UTC m=+0.440026528 container attach 63cfff8a7a451f56dc83732947be7b2b29540adc2c9a858bfa2023399a90e0bc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True) 2024-04-26T15:45:47.196 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:47 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:47.108+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:47.196 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:47 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:47.108+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.420292+0000 front 2024-04-26T15:45:19.420331+0000 (oldest deadline 2024-04-26T15:45:42.320017+0000) 2024-04-26T15:45:47.196 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:47 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:47.093+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:47.196 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:47 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:47.093+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:47.448 INFO:journalctl@ceph.osd.4.smithi164.stdout:Apr 26 15:45:47 smithi164 podman[75372]: 2024-04-26 15:45:47.412719777 +0000 UTC m=+0.656003848 container died 63cfff8a7a451f56dc83732947be7b2b29540adc2c9a858bfa2023399a90e0bc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_CLEAN=True) 2024-04-26T15:45:47.448 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:47 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:47.422+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:47.448 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:47 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:47.422+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:48.063 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.4.service' 2024-04-26T15:45:48.100 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T15:45:48.100 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-26T15:45:48.100 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-26T15:45:48.100 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.5 2024-04-26T15:45:48.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:48 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:48.101+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:48.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:48 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:48.101+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.420292+0000 front 2024-04-26T15:45:19.420331+0000 (oldest deadline 2024-04-26T15:45:42.320017+0000) 2024-04-26T15:45:48.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:48 smithi164 systemd[1]: Stopping Ceph osd.5 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:45:48.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:48 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:48.121+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:48.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:48 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:48.121+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:48.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:48 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:48.462+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:48.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:48 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:48.462+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:48.848 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:48 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:48.452+0000 7f8680210700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:45:48.848 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:48 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:48.452+0000 7f8680210700 -1 osd.5 154 *** Got signal Terminated *** 2024-04-26T15:45:48.848 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:48 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:48.452+0000 7f8680210700 -1 osd.5 154 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:45:49.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:49 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:49.053+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:49.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:49 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:49.053+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.420292+0000 front 2024-04-26T15:45:19.420331+0000 (oldest deadline 2024-04-26T15:45:42.320017+0000) 2024-04-26T15:45:49.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:49 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:49.123+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:49.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:49 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:49.123+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:49.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:49 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:49.434+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:49.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:49 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:49.434+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:50.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:50 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:50.044+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:50.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:50 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:50.044+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.420292+0000 front 2024-04-26T15:45:19.420331+0000 (oldest deadline 2024-04-26T15:45:42.320017+0000) 2024-04-26T15:45:50.348 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:50 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:50.044+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:29.222053+0000 front 2024-04-26T15:45:29.222065+0000 (oldest deadline 2024-04-26T15:45:49.721993+0000) 2024-04-26T15:45:50.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:50 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:50.143+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:50.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:50 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:50.143+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:50.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:50 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:50.474+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:50.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:50 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:50.474+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:50.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:50 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:50.474+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:24.355846+0000 front 2024-04-26T15:45:24.355859+0000 (oldest deadline 2024-04-26T15:45:50.255783+0000) 2024-04-26T15:45:51.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:51 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:51.069+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:51.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:51 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:51.069+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.420292+0000 front 2024-04-26T15:45:19.420331+0000 (oldest deadline 2024-04-26T15:45:42.320017+0000) 2024-04-26T15:45:51.347 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:51 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:51.069+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:29.222053+0000 front 2024-04-26T15:45:29.222065+0000 (oldest deadline 2024-04-26T15:45:49.721993+0000) 2024-04-26T15:45:51.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:51 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:51.151+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:51.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:51 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:51.151+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:51.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:51 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:51.427+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:51.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:51 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:51.427+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:51.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:51 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:51.427+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:24.355846+0000 front 2024-04-26T15:45:24.355859+0000 (oldest deadline 2024-04-26T15:45:50.255783+0000) 2024-04-26T15:45:52.451 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:52 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:52.103+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:52.451 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:52 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:52.103+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.420292+0000 front 2024-04-26T15:45:19.420331+0000 (oldest deadline 2024-04-26T15:45:42.320017+0000) 2024-04-26T15:45:52.452 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:52 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:52.103+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:29.222053+0000 front 2024-04-26T15:45:29.222065+0000 (oldest deadline 2024-04-26T15:45:49.721993+0000) 2024-04-26T15:45:52.452 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:52 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:52.168+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:52.452 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:52 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:52.168+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:52.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:52 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:52.452+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:52.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:52 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:52.452+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:52.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:52 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:52.452+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:24.355846+0000 front 2024-04-26T15:45:24.355859+0000 (oldest deadline 2024-04-26T15:45:50.255783+0000) 2024-04-26T15:45:53.448 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:53.129+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:13.218396+0000 front 2024-04-26T15:45:13.218331+0000 (oldest deadline 2024-04-26T15:45:33.718127+0000) 2024-04-26T15:45:53.448 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:53.129+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.420292+0000 front 2024-04-26T15:45:19.420331+0000 (oldest deadline 2024-04-26T15:45:42.320017+0000) 2024-04-26T15:45:53.448 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5[57193]: 2024-04-26T15:45:53.129+0000 7f86787fc700 -1 osd.5 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:29.222053+0000 front 2024-04-26T15:45:29.222065+0000 (oldest deadline 2024-04-26T15:45:49.721993+0000) 2024-04-26T15:45:53.448 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:53.213+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:53.448 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:53.213+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:53.449 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:53.213+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:45:53.819 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:53.448+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:53.819 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:53.448+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:53.819 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:53 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:53.448+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:24.355846+0000 front 2024-04-26T15:45:24.355859+0000 (oldest deadline 2024-04-26T15:45:50.255783+0000) 2024-04-26T15:45:54.097 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:53 smithi164 podman[75488]: 2024-04-26 15:45:53.820371771 +0000 UTC m=+5.473694049 container died 61be8ee4a0ab84c7830590a8eafe214d30dcaf681afa960b732f5b0340d8ccf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T15:45:54.597 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:54 smithi164 podman[75488]: 2024-04-26 15:45:54.419976213 +0000 UTC m=+6.073298479 container cleanup 61be8ee4a0ab84c7830590a8eafe214d30dcaf681afa960b732f5b0340d8ccf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-26T15:45:54.598 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:54 smithi164 bash[75488]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5 2024-04-26T15:45:54.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:54 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:54.205+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:54.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:54 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:54.205+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:54.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:54 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:54.205+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:45:54.599 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:54 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:54.407+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:54.599 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:54 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:54.407+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:54.599 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:54 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:54.407+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:24.355846+0000 front 2024-04-26T15:45:24.355859+0000 (oldest deadline 2024-04-26T15:45:50.255783+0000) 2024-04-26T15:45:54.990 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:54 smithi164 podman[75506]: 2024-04-26 15:45:54.662058151 +0000 UTC m=+0.837094432 container remove 61be8ee4a0ab84c7830590a8eafe214d30dcaf681afa960b732f5b0340d8ccf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T15:45:55.242 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:55 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:55.212+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:55.243 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:55 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:55.213+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:55.243 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:55 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:55.213+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:45:55.243 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:54 smithi164 podman[75596]: 2024-04-26 15:45:54.894586713 +0000 UTC m=+0.017908296 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:45:55.243 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:55 smithi164 podman[75596]: 2024-04-26 15:45:55.012860826 +0000 UTC m=+0.136182395 container create 243b20dc3a7f0850e6bdff31f6e058f8b9c7f14a2bdbf1bf53cf368561bf3c9d (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-26T15:45:55.243 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:55 smithi164 podman[75596]: 2024-04-26 15:45:55.240593759 +0000 UTC m=+0.363915338 container init 243b20dc3a7f0850e6bdff31f6e058f8b9c7f14a2bdbf1bf53cf368561bf3c9d (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-04-26T15:45:55.243 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:55 smithi164 podman[75596]: 2024-04-26 15:45:55.244093431 +0000 UTC m=+0.367415002 container start 243b20dc3a7f0850e6bdff31f6e058f8b9c7f14a2bdbf1bf53cf368561bf3c9d (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-26T15:45:55.546 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:55 smithi164 podman[75596]: 2024-04-26 15:45:55.34041129 +0000 UTC m=+0.463732863 container attach 243b20dc3a7f0850e6bdff31f6e058f8b9c7f14a2bdbf1bf53cf368561bf3c9d (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T15:45:55.546 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:55 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:55.435+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:55.546 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:55 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:55.435+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:55.546 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:55 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:55.435+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:24.355846+0000 front 2024-04-26T15:45:24.355859+0000 (oldest deadline 2024-04-26T15:45:50.255783+0000) 2024-04-26T15:45:55.847 INFO:journalctl@ceph.osd.5.smithi164.stdout:Apr 26 15:45:55 smithi164 podman[75596]: 2024-04-26 15:45:55.547457302 +0000 UTC m=+0.670778873 container died 243b20dc3a7f0850e6bdff31f6e058f8b9c7f14a2bdbf1bf53cf368561bf3c9d (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-26T15:45:56.174 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.5.service' 2024-04-26T15:45:56.215 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T15:45:56.215 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-26T15:45:56.215 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-26T15:45:56.215 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.6 2024-04-26T15:45:56.457 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:56 smithi164 systemd[1]: Stopping Ceph osd.6 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:45:56.457 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:56 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:56.211+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:56.457 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:56 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:56.211+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:56.457 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:56 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:56.211+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:45:56.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:56 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:56.457+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:56.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:56 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:56.457+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:56.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:56 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:56.457+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:24.355846+0000 front 2024-04-26T15:45:24.355859+0000 (oldest deadline 2024-04-26T15:45:50.255783+0000) 2024-04-26T15:45:56.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:56 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:56.568+0000 7f934aad9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:45:56.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:56 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:56.568+0000 7f934aad9700 -1 osd.6 154 *** Got signal Terminated *** 2024-04-26T15:45:56.848 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:56 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:56.568+0000 7f934aad9700 -1 osd.6 154 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:45:57.506 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:57 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:57.176+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:57.506 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:57 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:57.176+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:57.506 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:57 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:57.176+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:45:57.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:57 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:57.506+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:57.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:57 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:57.506+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:57.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:57 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:57.506+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:24.355846+0000 front 2024-04-26T15:45:24.355859+0000 (oldest deadline 2024-04-26T15:45:50.255783+0000) 2024-04-26T15:45:58.485 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:58 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:58.186+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:58.486 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:58 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:58.186+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:58.486 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:58 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:58.186+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:45:58.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:58 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:58.520+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:58.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:58 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:58.520+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:58.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:58 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:58.520+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:24.355846+0000 front 2024-04-26T15:45:24.355859+0000 (oldest deadline 2024-04-26T15:45:50.255783+0000) 2024-04-26T15:45:59.542 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:59 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:59.200+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:45:59.542 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:59 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:59.200+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:45:59.542 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:59 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:59.200+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:45:59.542 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:45:59 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:45:59.200+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6830 osd.3 since back 2024-04-26T15:45:33.593226+0000 front 2024-04-26T15:45:33.593246+0000 (oldest deadline 2024-04-26T15:45:58.293106+0000) 2024-04-26T15:45:59.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:59 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:59.542+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:45:59.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:59 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:59.542+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:45:59.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:45:59 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:45:59.542+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:24.355846+0000 front 2024-04-26T15:45:24.355859+0000 (oldest deadline 2024-04-26T15:45:50.255783+0000) 2024-04-26T15:46:00.536 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:00 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:00.186+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:46:00.536 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:00 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:00.186+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:46:00.536 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:00 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:00.186+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:46:00.536 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:00 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:00.186+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6830 osd.3 since back 2024-04-26T15:45:33.593226+0000 front 2024-04-26T15:45:33.593246+0000 (oldest deadline 2024-04-26T15:45:58.293106+0000) 2024-04-26T15:46:00.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:00 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:46:00.536+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:11.654066+0000 front 2024-04-26T15:45:11.654238+0000 (oldest deadline 2024-04-26T15:45:34.554068+0000) 2024-04-26T15:46:00.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:00 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:46:00.536+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:20.255722+0000 front 2024-04-26T15:45:20.255703+0000 (oldest deadline 2024-04-26T15:45:44.355352+0000) 2024-04-26T15:46:00.847 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:00 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6[59477]: 2024-04-26T15:46:00.536+0000 7f93470cd700 -1 osd.6 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:24.355846+0000 front 2024-04-26T15:45:24.355859+0000 (oldest deadline 2024-04-26T15:45:50.255783+0000) 2024-04-26T15:46:01.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:01 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:01.234+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:46:01.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:01 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:01.234+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:46:01.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:01 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:01.234+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:46:01.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:01 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:01.234+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6830 osd.3 since back 2024-04-26T15:45:33.593226+0000 front 2024-04-26T15:45:33.593246+0000 (oldest deadline 2024-04-26T15:45:58.293106+0000) 2024-04-26T15:46:02.434 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:02 smithi164 podman[75703]: 2024-04-26 15:46:02.178285914 +0000 UTC m=+5.733520119 container died 78ef3c811a10785f20696f954c9884235e247393a97d1bda3041699372f071da (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T15:46:02.435 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:02 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:02.252+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:46:02.435 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:02 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:02.252+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:46:02.435 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:02 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:02.252+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:46:02.435 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:02 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:02.252+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6830 osd.3 since back 2024-04-26T15:45:33.593226+0000 front 2024-04-26T15:45:33.593246+0000 (oldest deadline 2024-04-26T15:45:58.293106+0000) 2024-04-26T15:46:02.835 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:02 smithi164 podman[75703]: 2024-04-26 15:46:02.618563357 +0000 UTC m=+6.173797599 container cleanup 78ef3c811a10785f20696f954c9884235e247393a97d1bda3041699372f071da (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-26T15:46:02.835 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:02 smithi164 bash[75703]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6 2024-04-26T15:46:03.097 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:02 smithi164 podman[75721]: 2024-04-26 15:46:02.836624282 +0000 UTC m=+0.659903268 container remove 78ef3c811a10785f20696f954c9884235e247393a97d1bda3041699372f071da (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T15:46:03.513 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:03 smithi164 podman[75811]: 2024-04-26 15:46:03.186844983 +0000 UTC m=+0.017729984 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:46:03.513 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:03 smithi164 podman[75811]: 2024-04-26 15:46:03.304050478 +0000 UTC m=+0.134935473 container create 315fa58367d0a9a830b1965ac84b9359d8b027b578371bf215f419e610c3648b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-26T15:46:03.514 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:03 smithi164 podman[75811]: 2024-04-26 15:46:03.514791679 +0000 UTC m=+0.345676678 container init 315fa58367d0a9a830b1965ac84b9359d8b027b578371bf215f419e610c3648b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, ceph=True, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-26T15:46:03.514 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:03 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:03.204+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:46:03.514 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:03 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:03.204+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:46:03.514 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:03 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:03.204+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:46:03.514 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:03 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:03.204+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6830 osd.3 since back 2024-04-26T15:45:33.593226+0000 front 2024-04-26T15:45:33.593246+0000 (oldest deadline 2024-04-26T15:45:58.293106+0000) 2024-04-26T15:46:03.813 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:03 smithi164 podman[75811]: 2024-04-26 15:46:03.518918682 +0000 UTC m=+0.349803677 container start 315fa58367d0a9a830b1965ac84b9359d8b027b578371bf215f419e610c3648b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-26T15:46:03.813 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:03 smithi164 podman[75811]: 2024-04-26 15:46:03.606932152 +0000 UTC m=+0.437817163 container attach 315fa58367d0a9a830b1965ac84b9359d8b027b578371bf215f419e610c3648b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-26T15:46:04.097 INFO:journalctl@ceph.osd.6.smithi164.stdout:Apr 26 15:46:03 smithi164 podman[75811]: 2024-04-26 15:46:03.814685975 +0000 UTC m=+0.645570981 container died 315fa58367d0a9a830b1965ac84b9359d8b027b578371bf215f419e610c3648b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T15:46:04.412 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:04 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:04.222+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:46:04.413 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:04 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:04.222+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:46:04.413 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:04 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:04.222+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:46:04.413 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:04 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:04.222+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6830 osd.3 since back 2024-04-26T15:45:33.593226+0000 front 2024-04-26T15:45:33.593246+0000 (oldest deadline 2024-04-26T15:45:58.293106+0000) 2024-04-26T15:46:04.559 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.6.service' 2024-04-26T15:46:04.597 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T15:46:04.597 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-26T15:46:04.597 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-26T15:46:04.597 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.7 2024-04-26T15:46:05.097 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:04 smithi164 systemd[1]: Stopping Ceph osd.7 for 71e28ad8-03e0-11ef-bc93-c7b262605968... 2024-04-26T15:46:05.097 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:04 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:04.920+0000 7f6ce5315700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:46:05.097 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:04 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:04.920+0000 7f6ce5315700 -1 osd.7 154 *** Got signal Terminated *** 2024-04-26T15:46:05.097 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:04 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:04.920+0000 7f6ce5315700 -1 osd.7 154 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T15:46:05.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:05 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:05.263+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:46:05.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:05 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:05.263+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:46:05.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:05 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:05.264+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:46:05.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:05 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:05.264+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6830 osd.3 since back 2024-04-26T15:45:33.593226+0000 front 2024-04-26T15:45:33.593246+0000 (oldest deadline 2024-04-26T15:45:58.293106+0000) 2024-04-26T15:46:06.347 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:06 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:06.295+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:46:06.347 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:06 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:06.295+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:46:06.347 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:06 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:06.295+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:46:06.347 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:06 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:06.295+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6830 osd.3 since back 2024-04-26T15:45:33.593226+0000 front 2024-04-26T15:45:33.593246+0000 (oldest deadline 2024-04-26T15:45:58.293106+0000) 2024-04-26T15:46:07.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:07 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:07.259+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:46:07.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:07 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:07.259+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:46:07.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:07 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:07.259+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:46:07.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:07 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:07.259+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6830 osd.3 since back 2024-04-26T15:45:33.593226+0000 front 2024-04-26T15:45:33.593246+0000 (oldest deadline 2024-04-26T15:45:58.293106+0000) 2024-04-26T15:46:08.597 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:08 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:08.242+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:46:08.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:08 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:08.242+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:46:08.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:08 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:08.242+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:46:08.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:08 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:08.242+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6830 osd.3 since back 2024-04-26T15:45:33.593226+0000 front 2024-04-26T15:45:33.593246+0000 (oldest deadline 2024-04-26T15:45:58.293106+0000) 2024-04-26T15:46:08.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:08 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:08.242+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.164:6804 osd.4 since back 2024-04-26T15:45:43.494360+0000 front 2024-04-26T15:45:43.494524+0000 (oldest deadline 2024-04-26T15:46:08.194386+0000) 2024-04-26T15:46:09.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:09 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:09.281+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6806 osd.0 since back 2024-04-26T15:45:09.789470+0000 front 2024-04-26T15:45:09.789555+0000 (oldest deadline 2024-04-26T15:45:35.089329+0000) 2024-04-26T15:46:09.598 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:09 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:09.281+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6814 osd.1 since back 2024-04-26T15:45:19.091154+0000 front 2024-04-26T15:45:19.091089+0000 (oldest deadline 2024-04-26T15:45:44.390733+0000) 2024-04-26T15:46:09.599 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:09 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:09.281+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6822 osd.2 since back 2024-04-26T15:45:28.992359+0000 front 2024-04-26T15:45:28.992394+0000 (oldest deadline 2024-04-26T15:45:52.492258+0000) 2024-04-26T15:46:09.599 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:09 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:09.281+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.112:6830 osd.3 since back 2024-04-26T15:45:33.593226+0000 front 2024-04-26T15:45:33.593246+0000 (oldest deadline 2024-04-26T15:45:58.293106+0000) 2024-04-26T15:46:09.599 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:09 smithi164 ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7[61744]: 2024-04-26T15:46:09.281+0000 7f6cdd901700 -1 osd.7 154 heartbeat_check: no reply from 172.21.15.164:6804 osd.4 since back 2024-04-26T15:45:43.494360+0000 front 2024-04-26T15:45:43.494524+0000 (oldest deadline 2024-04-26T15:46:08.194386+0000) 2024-04-26T15:46:10.754 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:10 smithi164 podman[75948]: 2024-04-26 15:46:10.503991781 +0000 UTC m=+5.689517758 container died 955dfe6c53cec0539a3b6d8156bc7e09ce31b824cf277822406b46caa7f59c0f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=) 2024-04-26T15:46:11.097 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:10 smithi164 podman[75948]: 2024-04-26 15:46:10.929586649 +0000 UTC m=+6.115112643 container cleanup 955dfe6c53cec0539a3b6d8156bc7e09ce31b824cf277822406b46caa7f59c0f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424) 2024-04-26T15:46:11.098 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:10 smithi164 bash[75948]: ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7 2024-04-26T15:46:11.544 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:11 smithi164 podman[75977]: 2024-04-26 15:46:11.122829669 +0000 UTC m=+0.619724458 container remove 955dfe6c53cec0539a3b6d8156bc7e09ce31b824cf277822406b46caa7f59c0f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-26T15:46:11.847 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:11 smithi164 podman[76067]: 2024-04-26 15:46:11.448924835 +0000 UTC m=+0.018292624 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T15:46:11.847 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:11 smithi164 podman[76067]: 2024-04-26 15:46:11.58212343 +0000 UTC m=+0.151491262 container create db270f3a5e20e5d577a3cdb99690390824b682474e3e6ab5f3dec84ded8fda48 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T15:46:11.847 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:11 smithi164 podman[76067]: 2024-04-26 15:46:11.792816206 +0000 UTC m=+0.362183997 container init db270f3a5e20e5d577a3cdb99690390824b682474e3e6ab5f3dec84ded8fda48 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-26T15:46:11.847 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:11 smithi164 podman[76067]: 2024-04-26 15:46:11.795804141 +0000 UTC m=+0.365171933 container start db270f3a5e20e5d577a3cdb99690390824b682474e3e6ab5f3dec84ded8fda48 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-26T15:46:12.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:11 smithi164 podman[76067]: 2024-04-26 15:46:11.892710291 +0000 UTC m=+0.462078090 container attach db270f3a5e20e5d577a3cdb99690390824b682474e3e6ab5f3dec84ded8fda48 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True) 2024-04-26T15:46:12.348 INFO:journalctl@ceph.osd.7.smithi164.stdout:Apr 26 15:46:12 smithi164 podman[76067]: 2024-04-26 15:46:12.093275657 +0000 UTC m=+0.662643448 container died db270f3a5e20e5d577a3cdb99690390824b682474e3e6ab5f3dec84ded8fda48 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-71e28ad8-03e0-11ef-bc93-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-26T15:46:12.650 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-71e28ad8-03e0-11ef-bc93-c7b262605968@osd.7.service' 2024-04-26T15:46:12.688 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T15:46:12.688 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-26T15:46:12.688 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T15:49:12.075 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T15:49:31.861 DEBUG:teuthology.orchestra.run.smithi112:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T15:49:31.894 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T15:49:31.923 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-26T15:49:31.925 DEBUG:teuthology.misc:Transferring archived files from smithi112:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674785/remote/smithi112/crash 2024-04-26T15:49:31.926 DEBUG:teuthology.orchestra.run.smithi112:> sudo tar c -f - -C /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/crash -- . 2024-04-26T15:49:31.972 INFO:teuthology.orchestra.run.smithi112.stderr:tar: /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T15:49:31.972 INFO:teuthology.orchestra.run.smithi112.stderr:tar: Error is not recoverable: exiting now 2024-04-26T15:49:31.974 DEBUG:teuthology.misc:Transferring archived files from smithi164:/var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674785/remote/smithi164/crash 2024-04-26T15:49:31.974 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/crash -- . 2024-04-26T15:49:32.009 INFO:teuthology.orchestra.run.smithi164.stderr:tar: /var/lib/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T15:49:32.009 INFO:teuthology.orchestra.run.smithi164.stderr:tar: Error is not recoverable: exiting now 2024-04-26T15:49:32.011 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-26T15:49:32.011 DEBUG:teuthology.orchestra.run.smithi112:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-26T15:49:32.041 INFO:tasks.cephadm:Compressing logs... 2024-04-26T15:49:32.041 DEBUG:teuthology.orchestra.run.smithi112:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T15:49:32.084 DEBUG:teuthology.orchestra.run.smithi164:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T15:49:32.106 INFO:teuthology.orchestra.run.smithi112.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T15:49:32.106 INFO:teuthology.orchestra.run.smithi112.stderr:: No such file or directory 2024-04-26T15:49:32.106 INFO:teuthology.orchestra.run.smithi112.stderr:gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mon.smithi112.log 2024-04-26T15:49:32.107 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.log 2024-04-26T15:49:32.107 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mon.smithi112.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi112.fqmway.log 2024-04-26T15:49:32.110 INFO:teuthology.orchestra.run.smithi112.stderr:gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-26T15:49:32.110 INFO:teuthology.orchestra.run.smithi112.stderr:gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T15:49:32.110 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.log: /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi112.fqmway.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-26T15:49:32.110 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.2.log: /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mgr.smithi112.llioug.log 2024-04-26T15:49:32.111 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-26T15:49:32.114 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mgr.smithi112.llioug.log: 87.1% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T15:49:32.114 INFO:teuthology.orchestra.run.smithi112.stderr: 91.6% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T15:49:32.114 INFO:teuthology.orchestra.run.smithi112.stderr:gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-26T15:49:32.114 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:: No such file or directory 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi164.gbxewm.log 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.log 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi164.gbxewm.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi164.gbxewm.log.gz 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.audit.log: /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mon.smithi164.log 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.cephadm.log: 85.7% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T15:49:32.116 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mgr.smithi164.tuhwus.log 2024-04-26T15:49:32.117 INFO:teuthology.orchestra.run.smithi164.stderr: 87.1% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T15:49:32.117 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mon.smithi164.log: 91.7% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T15:49:32.117 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T15:49:32.117 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mgr.smithi164.tuhwus.log: /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-volume.log: 90.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T15:49:32.128 INFO:teuthology.orchestra.run.smithi164.stderr: 91.2% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T15:49:32.132 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.0.log: 91.5% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi112.fqmway.log.gz 2024-04-26T15:49:32.132 INFO:teuthology.orchestra.run.smithi112.stderr:gzip -5 --verbose -- /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T15:49:32.133 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.cephadm.log: 85.8% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T15:49:32.146 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-volume.log: 91.5% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T15:49:32.149 INFO:teuthology.orchestra.run.smithi112.stderr: 91.2% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T15:49:32.158 INFO:teuthology.orchestra.run.smithi164.stderr: 89.7% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mgr.smithi164.tuhwus.log.gz 2024-04-26T15:49:32.413 INFO:teuthology.orchestra.run.smithi112.stderr: 88.4% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mgr.smithi112.llioug.log.gz 2024-04-26T15:49:32.470 INFO:teuthology.orchestra.run.smithi164.stderr: 92.7% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mon.smithi164.log.gz 2024-04-26T15:49:34.076 INFO:teuthology.orchestra.run.smithi112.stderr: 92.0% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-mon.smithi112.log.gz 2024-04-26T15:49:46.126 INFO:teuthology.orchestra.run.smithi112.stderr: 94.5% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-26T15:49:46.923 INFO:teuthology.orchestra.run.smithi164.stderr: 93.9% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-26T15:49:49.015 INFO:teuthology.orchestra.run.smithi164.stderr: 94.3% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-26T15:49:49.604 INFO:teuthology.orchestra.run.smithi112.stderr: 94.1% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-26T15:49:51.248 INFO:teuthology.orchestra.run.smithi112.stderr: 94.2% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-26T15:49:51.464 INFO:teuthology.orchestra.run.smithi112.stderr: 94.1% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-26T15:49:51.467 INFO:teuthology.orchestra.run.smithi112.stderr: 2024-04-26T15:49:51.467 INFO:teuthology.orchestra.run.smithi112.stderr:real 0m19.371s 2024-04-26T15:49:51.467 INFO:teuthology.orchestra.run.smithi112.stderr:user 1m9.492s 2024-04-26T15:49:51.467 INFO:teuthology.orchestra.run.smithi112.stderr:sys 0m2.724s 2024-04-26T15:49:51.882 INFO:teuthology.orchestra.run.smithi164.stderr: 94.1% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-26T15:49:55.105 INFO:teuthology.orchestra.run.smithi164.stderr: 94.1% -- replaced with /var/log/ceph/71e28ad8-03e0-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-26T15:49:55.107 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-04-26T15:49:55.107 INFO:teuthology.orchestra.run.smithi164.stderr:real 0m23.010s 2024-04-26T15:49:55.107 INFO:teuthology.orchestra.run.smithi164.stderr:user 1m11.957s 2024-04-26T15:49:55.107 INFO:teuthology.orchestra.run.smithi164.stderr:sys 0m2.824s 2024-04-26T15:49:55.107 INFO:tasks.cephadm:Archiving logs... 2024-04-26T15:49:55.108 DEBUG:teuthology.misc:Transferring archived files from smithi112:/var/log/ceph to /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674785/remote/smithi112/log 2024-04-26T15:49:55.109 DEBUG:teuthology.orchestra.run.smithi112:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T15:50:04.737 DEBUG:teuthology.misc:Transferring archived files from smithi164:/var/log/ceph to /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674785/remote/smithi164/log 2024-04-26T15:50:04.738 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T15:50:15.204 INFO:tasks.cephadm:Removing cluster... 2024-04-26T15:50:15.205 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 --force 2024-04-26T15:50:15.864 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 71e28ad8-03e0-11ef-bc93-c7b262605968 --force 2024-04-26T15:50:16.252 INFO:tasks.cephadm:Removing cephadm ... 2024-04-26T15:50:16.252 DEBUG:teuthology.orchestra.run.smithi112:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T15:50:16.267 DEBUG:teuthology.orchestra.run.smithi164:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T15:50:16.282 INFO:tasks.cephadm:Teardown complete 2024-04-26T15:50:16.283 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-26T15:50:16.294 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-26T15:50:16.295 DEBUG:teuthology.orchestra.run.smithi112:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T15:50:16.309 DEBUG:teuthology.orchestra.run.smithi164:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T15:50:16.322 INFO:teuthology.orchestra.run.smithi112.stderr:bash: line 1: ntpq: command not found 2024-04-26T15:50:16.324 INFO:teuthology.orchestra.run.smithi112.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T15:50:16.324 INFO:teuthology.orchestra.run.smithi112.stdout:=============================================================================== 2024-04-26T15:50:16.325 INFO:teuthology.orchestra.run.smithi112.stdout:^* hv01.front.sepia.ceph.com 3 6 377 62 +2492us[+2537us] +/- 44ms 2024-04-26T15:50:16.325 INFO:teuthology.orchestra.run.smithi112.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 64 -464us[ -419us] +/- 76ms 2024-04-26T15:50:16.325 INFO:teuthology.orchestra.run.smithi112.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 63 -3587us[-3542us] +/- 74ms 2024-04-26T15:50:16.325 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:50:16.336 INFO:teuthology.orchestra.run.smithi164.stderr:bash: line 1: ntpq: command not found 2024-04-26T15:50:16.340 INFO:teuthology.orchestra.run.smithi164.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T15:50:16.340 INFO:teuthology.orchestra.run.smithi164.stdout:=============================================================================== 2024-04-26T15:50:16.340 INFO:teuthology.orchestra.run.smithi164.stdout:^* hv01.front.sepia.ceph.com 3 6 377 1 +2451us[+2496us] +/- 45ms 2024-04-26T15:50:16.340 INFO:teuthology.orchestra.run.smithi164.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 1 -466us[ -466us] +/- 77ms 2024-04-26T15:50:16.340 INFO:teuthology.orchestra.run.smithi164.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 63 -3596us[-3570us] +/- 74ms 2024-04-26T15:50:16.340 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:50:16.341 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-26T15:50:16.352 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-26T15:50:16.353 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-26T15:50:16.363 DEBUG:teuthology.orchestra.run.smithi112:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T15:50:16.439 DEBUG:teuthology.orchestra.run.smithi164:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T15:50:16.509 DEBUG:teuthology.orchestra.run.smithi112:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-26T15:50:16.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T15:50:16.536 DEBUG:teuthology.orchestra.run.smithi164:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-26T15:50:16.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T15:50:16.564 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-26T15:50:16.578 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-26T15:50:16.588 INFO:teuthology.task.internal:Duration was 2026.941529 seconds 2024-04-26T15:50:16.589 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-26T15:50:16.600 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-26T15:50:16.600 DEBUG:teuthology.orchestra.run.smithi112:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T15:50:16.603 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T15:50:16.636 INFO:teuthology.orchestra.run.smithi112.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T15:50:16.639 INFO:teuthology.orchestra.run.smithi164.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T15:50:16.896 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-26T15:50:16.896 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi112.front.sepia.ceph.com 2024-04-26T15:50:16.897 DEBUG:teuthology.orchestra.run.smithi112:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T15:50:16.916 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi164.front.sepia.ceph.com 2024-04-26T15:50:16.916 DEBUG:teuthology.orchestra.run.smithi164:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T15:50:16.962 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-26T15:50:16.963 DEBUG:teuthology.orchestra.run.smithi112:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T15:50:16.965 DEBUG:teuthology.orchestra.run.smithi164:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T15:50:17.028 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-26T15:50:17.028 DEBUG:teuthology.orchestra.run.smithi112:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T15:50:17.030 DEBUG:teuthology.orchestra.run.smithi164:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T15:50:30.611 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-26T15:50:30.623 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-26T15:50:30.624 DEBUG:teuthology.orchestra.run.smithi112:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T15:50:30.647 DEBUG:teuthology.orchestra.run.smithi164:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T15:50:30.676 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-26T15:50:30.688 DEBUG:teuthology.orchestra.run.smithi112:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T15:50:30.693 DEBUG:teuthology.orchestra.run.smithi164:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T15:50:30.710 INFO:teuthology.orchestra.run.smithi112.stdout:kernel.core_pattern = core 2024-04-26T15:50:30.743 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern = core 2024-04-26T15:50:30.769 DEBUG:teuthology.orchestra.run.smithi112:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T15:50:30.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T15:50:30.791 DEBUG:teuthology.orchestra.run.smithi164:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T15:50:30.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T15:50:30.825 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-26T15:50:30.837 INFO:teuthology.task.internal:Transferring archived files... 2024-04-26T15:50:30.838 DEBUG:teuthology.misc:Transferring archived files from smithi112:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674785/remote/smithi112 2024-04-26T15:50:30.838 DEBUG:teuthology.orchestra.run.smithi112:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T15:50:30.897 DEBUG:teuthology.misc:Transferring archived files from smithi164:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674785/remote/smithi164 2024-04-26T15:50:30.897 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T15:50:30.939 INFO:teuthology.task.internal:Removing archive directory... 2024-04-26T15:50:30.939 DEBUG:teuthology.orchestra.run.smithi112:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T15:50:30.942 DEBUG:teuthology.orchestra.run.smithi164:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T15:50:30.988 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-26T15:50:31.008 INFO:teuthology.task.internal:Not uploading archives. 2024-04-26T15:50:31.008 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-26T15:50:31.025 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-26T15:50:31.025 DEBUG:teuthology.orchestra.run.smithi112:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T15:50:31.028 DEBUG:teuthology.orchestra.run.smithi164:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T15:50:31.039 INFO:teuthology.orchestra.run.smithi112.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 15:50 /home/ubuntu/cephtest 2024-04-26T15:50:31.043 INFO:teuthology.orchestra.run.smithi164.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 15:50 /home/ubuntu/cephtest 2024-04-26T15:50:31.045 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-26T15:50:31.064 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-26T15:50:31.097 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2026.9415292739868 owner: scheduled_phlogistonjohn@teuthology success: true 2024-04-26T15:50:31.098 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T15:50:31.186 INFO:teuthology.run:pass