2024-04-22T14:58:38.138 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-22T14:58:38.138 DEBUG:teuthology.run:Teuthology command: teuthology --name adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi --archive /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668325 --verbose --description orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --owner scheduled_adking@teuthology -- /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668325/orig.config.yaml 2024-04-22T14:58:38.176 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-22T14:58:38.258 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668325 branch: wip-adk-testing-2024-04-22-0809 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: adking@redhat.com first_in_suite: false job_id: '7668325' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk-testing-2024-04-22-0809 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 5883d494d3973b30b3f03e4a3f0e1e06e3c131aa ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 5883d494d3973b30b3f03e4a3f0e1e06e3c131aa selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-adk-testing-2024-04-22-0809 sha1: 5883d494d3973b30b3f03e4a3f0e1e06e3c131aa owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 523 sha1: 5883d494d3973b30b3f03e4a3f0e1e06e3c131aa sleep_before_teardown: 0 subset: 1/16 suite: orch:cephadm suite_branch: wip-adk-testing-2024-04-22-0809 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_5883d494d3973b30b3f03e4a3f0e1e06e3c131aa/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 5883d494d3973b30b3f03e4a3f0e1e06e3c131aa targets: smithi119.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFpweA75QB4u6PBbj5lOPK+FodfK9tUkzDC+yhGG/lTru2rjssYZlNyHRl5lmHmHsraNOwxuaSS0vRaz+Zb0bV8= smithi186.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPQ+nPLJsZiBeQbamcR/OQjPks2Ry35tZzkYQefJzmZ7dzhK8yKpT4X3p7bI8TayR85aCzTUJVNeQHALJtZOwm4= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-22_14:50:52 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-04-22T14:58:38.258 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_5883d494d3973b30b3f03e4a3f0e1e06e3c131aa/qa; will attempt to use it 2024-04-22T14:58:38.259 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_5883d494d3973b30b3f03e4a3f0e1e06e3c131aa/qa/tasks 2024-04-22T14:58:38.259 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-22T14:58:38.261 INFO:teuthology.task.internal:Checking packages... 2024-04-22T14:58:38.287 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '5883d494d3973b30b3f03e4a3f0e1e06e3c131aa' 2024-04-22T14:58:38.287 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-22T14:58:38.287 INFO:teuthology.packaging:ref: None 2024-04-22T14:58:38.287 INFO:teuthology.packaging:tag: None 2024-04-22T14:58:38.287 INFO:teuthology.packaging:branch: wip-adk-testing-2024-04-22-0809 2024-04-22T14:58:38.287 INFO:teuthology.packaging:sha1: 5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T14:58:38.287 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk-testing-2024-04-22-0809 2024-04-22T14:58:38.516 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3223.g5883d494 2024-04-22T14:58:38.517 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-22T14:58:38.522 INFO:teuthology.task.internal:no buildpackages task found 2024-04-22T14:58:38.522 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-22T14:58:38.539 INFO:teuthology.task.internal:Saving configuration 2024-04-22T14:58:38.554 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-22T14:58:38.560 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-22T14:58:38.585 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi119.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668325', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 14:52:11.799497', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFpweA75QB4u6PBbj5lOPK+FodfK9tUkzDC+yhGG/lTru2rjssYZlNyHRl5lmHmHsraNOwxuaSS0vRaz+Zb0bV8='} 2024-04-22T14:58:38.607 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi186.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668325', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 14:52:11.800707', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPQ+nPLJsZiBeQbamcR/OQjPks2Ry35tZzkYQefJzmZ7dzhK8yKpT4X3p7bI8TayR85aCzTUJVNeQHALJtZOwm4='} 2024-04-22T14:58:38.608 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-22T14:58:38.613 INFO:teuthology.task.internal:roles: ubuntu@smithi119.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-04-22T14:58:38.613 INFO:teuthology.task.internal:roles: ubuntu@smithi186.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-04-22T14:58:38.613 INFO:teuthology.run_tasks:Running task console_log... 2024-04-22T14:58:38.681 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f47f10ed160>, signals=[15]) 2024-04-22T14:58:38.682 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-22T14:58:38.687 INFO:teuthology.task.internal:Opening connections... 2024-04-22T14:58:38.687 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T14:58:38.689 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi119.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T14:58:38.765 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi186.front.sepia.ceph.com 2024-04-22T14:58:38.766 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi186.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T14:58:38.843 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-22T14:58:38.849 DEBUG:teuthology.orchestra.run.smithi119:> uname -m 2024-04-22T14:58:38.866 INFO:teuthology.orchestra.run.smithi119.stdout:x86_64 2024-04-22T14:58:38.867 DEBUG:teuthology.orchestra.run.smithi119:> cat /etc/os-release 2024-04-22T14:58:38.924 INFO:teuthology.orchestra.run.smithi119.stdout:NAME="CentOS Stream" 2024-04-22T14:58:38.924 INFO:teuthology.orchestra.run.smithi119.stdout:VERSION="9" 2024-04-22T14:58:38.924 INFO:teuthology.orchestra.run.smithi119.stdout:ID="centos" 2024-04-22T14:58:38.924 INFO:teuthology.orchestra.run.smithi119.stdout:ID_LIKE="rhel fedora" 2024-04-22T14:58:38.924 INFO:teuthology.orchestra.run.smithi119.stdout:VERSION_ID="9" 2024-04-22T14:58:38.924 INFO:teuthology.orchestra.run.smithi119.stdout:PLATFORM_ID="platform:el9" 2024-04-22T14:58:38.924 INFO:teuthology.orchestra.run.smithi119.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-22T14:58:38.924 INFO:teuthology.orchestra.run.smithi119.stdout:ANSI_COLOR="0;31" 2024-04-22T14:58:38.924 INFO:teuthology.orchestra.run.smithi119.stdout:LOGO="fedora-logo-icon" 2024-04-22T14:58:38.925 INFO:teuthology.orchestra.run.smithi119.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-22T14:58:38.925 INFO:teuthology.orchestra.run.smithi119.stdout:HOME_URL="https://centos.org/" 2024-04-22T14:58:38.925 INFO:teuthology.orchestra.run.smithi119.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-22T14:58:38.925 INFO:teuthology.orchestra.run.smithi119.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-22T14:58:38.925 INFO:teuthology.orchestra.run.smithi119.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-22T14:58:38.925 INFO:teuthology.lock.ops:Updating smithi119.front.sepia.ceph.com on lock server 2024-04-22T14:58:38.951 DEBUG:teuthology.orchestra.run.smithi186:> uname -m 2024-04-22T14:58:38.968 INFO:teuthology.orchestra.run.smithi186.stdout:x86_64 2024-04-22T14:58:38.969 DEBUG:teuthology.orchestra.run.smithi186:> cat /etc/os-release 2024-04-22T14:58:39.025 INFO:teuthology.orchestra.run.smithi186.stdout:NAME="CentOS Stream" 2024-04-22T14:58:39.025 INFO:teuthology.orchestra.run.smithi186.stdout:VERSION="9" 2024-04-22T14:58:39.026 INFO:teuthology.orchestra.run.smithi186.stdout:ID="centos" 2024-04-22T14:58:39.026 INFO:teuthology.orchestra.run.smithi186.stdout:ID_LIKE="rhel fedora" 2024-04-22T14:58:39.026 INFO:teuthology.orchestra.run.smithi186.stdout:VERSION_ID="9" 2024-04-22T14:58:39.026 INFO:teuthology.orchestra.run.smithi186.stdout:PLATFORM_ID="platform:el9" 2024-04-22T14:58:39.026 INFO:teuthology.orchestra.run.smithi186.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-22T14:58:39.026 INFO:teuthology.orchestra.run.smithi186.stdout:ANSI_COLOR="0;31" 2024-04-22T14:58:39.026 INFO:teuthology.orchestra.run.smithi186.stdout:LOGO="fedora-logo-icon" 2024-04-22T14:58:39.026 INFO:teuthology.orchestra.run.smithi186.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-22T14:58:39.026 INFO:teuthology.orchestra.run.smithi186.stdout:HOME_URL="https://centos.org/" 2024-04-22T14:58:39.026 INFO:teuthology.orchestra.run.smithi186.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-22T14:58:39.026 INFO:teuthology.orchestra.run.smithi186.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-22T14:58:39.026 INFO:teuthology.orchestra.run.smithi186.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-22T14:58:39.027 INFO:teuthology.lock.ops:Updating smithi186.front.sepia.ceph.com on lock server 2024-04-22T14:58:39.051 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-22T14:58:39.059 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-22T14:58:39.064 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-22T14:58:39.064 DEBUG:teuthology.orchestra.run.smithi119:> test '!' -e /home/ubuntu/cephtest 2024-04-22T14:58:39.067 DEBUG:teuthology.orchestra.run.smithi186:> test '!' -e /home/ubuntu/cephtest 2024-04-22T14:58:39.082 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-22T14:58:39.094 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-22T14:58:39.094 DEBUG:teuthology.orchestra.run.smithi119:> test -z $(ls -A /var/lib/ceph) 2024-04-22T14:58:39.123 DEBUG:teuthology.orchestra.run.smithi186:> test -z $(ls -A /var/lib/ceph) 2024-04-22T14:58:39.228 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-22T14:58:39.281 INFO:teuthology.run_tasks:Running task kernel... 2024-04-22T14:58:39.299 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-22T14:58:39.299 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-22T14:58:39.299 DEBUG:teuthology.orchestra.run.smithi119:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-22T14:58:39.300 DEBUG:teuthology.orchestra.run.smithi186:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-22T14:58:39.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T14:58:39.316 DEBUG:teuthology.orchestra.run.smithi119:> uname -r 2024-04-22T14:58:39.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T14:58:39.317 DEBUG:teuthology.orchestra.run.smithi186:> uname -r 2024-04-22T14:58:39.373 INFO:teuthology.orchestra.run.smithi119.stdout:5.14.0-437.el9.x86_64 2024-04-22T14:58:39.373 INFO:teuthology.task.kernel:Running kernel on smithi119: 5.14.0-437.el9.x86_64 2024-04-22T14:58:39.373 DEBUG:teuthology.orchestra.run.smithi119:> sudo yum install -y kernel 2024-04-22T14:58:39.374 INFO:teuthology.orchestra.run.smithi186.stdout:5.14.0-437.el9.x86_64 2024-04-22T14:58:39.375 INFO:teuthology.task.kernel:Running kernel on smithi186: 5.14.0-437.el9.x86_64 2024-04-22T14:58:39.375 DEBUG:teuthology.orchestra.run.smithi186:> sudo yum install -y kernel 2024-04-22T14:58:41.742 INFO:teuthology.orchestra.run.smithi119.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.0 MB 00:00 2024-04-22T14:58:43.485 INFO:teuthology.orchestra.run.smithi119.stdout:CentOS Stream 9 - AppStream 28 MB/s | 19 MB 00:00 2024-04-22T14:58:48.196 INFO:teuthology.orchestra.run.smithi119.stdout:CentOS Stream 9 - CRB 5.8 MB/s | 6.2 MB 00:01 2024-04-22T14:58:49.937 INFO:teuthology.orchestra.run.smithi119.stdout:CentOS Stream 9 - Extras packages 40 kB/s | 16 kB 00:00 2024-04-22T14:58:50.383 INFO:teuthology.orchestra.run.smithi186.stdout:CentOS Stream 9 - BaseOS 887 kB/s | 8.0 MB 00:09 2024-04-22T14:58:51.621 INFO:teuthology.orchestra.run.smithi119.stdout:Extra Packages for Enterprise Linux 13 MB/s | 21 MB 00:01 2024-04-22T14:58:52.917 INFO:teuthology.orchestra.run.smithi186.stdout:CentOS Stream 9 - AppStream 13 MB/s | 19 MB 00:01 2024-04-22T14:58:56.754 INFO:teuthology.orchestra.run.smithi119.stdout:lab-extras 14 kB/s | 1.7 kB 00:00 2024-04-22T14:58:57.328 INFO:teuthology.orchestra.run.smithi186.stdout:CentOS Stream 9 - CRB 9.7 MB/s | 6.2 MB 00:00 2024-04-22T14:58:58.635 INFO:teuthology.orchestra.run.smithi119.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-22T14:58:58.636 INFO:teuthology.orchestra.run.smithi119.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-22T14:58:58.685 INFO:teuthology.orchestra.run.smithi186.stdout:CentOS Stream 9 - Extras packages 74 kB/s | 16 kB 00:00 2024-04-22T14:58:58.729 INFO:teuthology.orchestra.run.smithi119.stdout:Dependencies resolved. 2024-04-22T14:58:58.730 INFO:teuthology.orchestra.run.smithi119.stdout:Nothing to do. 2024-04-22T14:58:58.730 INFO:teuthology.orchestra.run.smithi119.stdout:Complete! 2024-04-22T14:58:58.817 DEBUG:teuthology.orchestra.run.smithi119:> echo no | sudo yum reinstall kernel || true 2024-04-22T14:58:59.321 INFO:teuthology.orchestra.run.smithi119.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 02:58:56 PM UTC. 2024-04-22T14:58:59.450 INFO:teuthology.orchestra.run.smithi119.stdout:Dependencies resolved. 2024-04-22T14:58:59.461 INFO:teuthology.orchestra.run.smithi119.stdout:================================================================================ 2024-04-22T14:58:59.461 INFO:teuthology.orchestra.run.smithi119.stdout: Package Architecture Version Repository Size 2024-04-22T14:58:59.461 INFO:teuthology.orchestra.run.smithi119.stdout:================================================================================ 2024-04-22T14:58:59.461 INFO:teuthology.orchestra.run.smithi119.stdout:Reinstalling: 2024-04-22T14:58:59.461 INFO:teuthology.orchestra.run.smithi119.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T14:58:59.462 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T14:58:59.462 INFO:teuthology.orchestra.run.smithi119.stdout:Transaction Summary 2024-04-22T14:58:59.462 INFO:teuthology.orchestra.run.smithi119.stdout:================================================================================ 2024-04-22T14:58:59.462 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T14:58:59.462 INFO:teuthology.orchestra.run.smithi119.stdout:Total download size: 255 k 2024-04-22T14:58:59.462 INFO:teuthology.orchestra.run.smithi119.stdout:Installed size: 0 2024-04-22T14:58:59.462 INFO:teuthology.orchestra.run.smithi119.stderr:Operation aborted. 2024-04-22T14:58:59.519 INFO:teuthology.orchestra.run.smithi119.stdout:Is this ok [y/N]: 2024-04-22T14:58:59.520 DEBUG:teuthology.orchestra.run.smithi119:> sudo yum reinstall -y kernel || true 2024-04-22T14:59:00.011 INFO:teuthology.orchestra.run.smithi186.stdout:Extra Packages for Enterprise Linux 17 MB/s | 21 MB 00:01 2024-04-22T14:59:00.013 INFO:teuthology.orchestra.run.smithi119.stdout:Last metadata expiration check: 0:00:04 ago on Mon 22 Apr 2024 02:58:56 PM UTC. 2024-04-22T14:59:00.129 INFO:teuthology.orchestra.run.smithi119.stdout:Dependencies resolved. 2024-04-22T14:59:00.130 INFO:teuthology.orchestra.run.smithi119.stdout:================================================================================ 2024-04-22T14:59:00.130 INFO:teuthology.orchestra.run.smithi119.stdout: Package Architecture Version Repository Size 2024-04-22T14:59:00.130 INFO:teuthology.orchestra.run.smithi119.stdout:================================================================================ 2024-04-22T14:59:00.130 INFO:teuthology.orchestra.run.smithi119.stdout:Reinstalling: 2024-04-22T14:59:00.130 INFO:teuthology.orchestra.run.smithi119.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T14:59:00.131 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T14:59:00.131 INFO:teuthology.orchestra.run.smithi119.stdout:Transaction Summary 2024-04-22T14:59:00.131 INFO:teuthology.orchestra.run.smithi119.stdout:================================================================================ 2024-04-22T14:59:00.131 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T14:59:00.131 INFO:teuthology.orchestra.run.smithi119.stdout:Total download size: 255 k 2024-04-22T14:59:00.131 INFO:teuthology.orchestra.run.smithi119.stdout:Installed size: 0 2024-04-22T14:59:00.131 INFO:teuthology.orchestra.run.smithi119.stdout:Downloading Packages: 2024-04-22T14:59:00.864 INFO:teuthology.orchestra.run.smithi119.stdout:kernel-5.14.0-437.el9.x86_64.rpm 597 kB/s | 255 kB 00:00 2024-04-22T14:59:00.865 INFO:teuthology.orchestra.run.smithi119.stdout:-------------------------------------------------------------------------------- 2024-04-22T14:59:00.865 INFO:teuthology.orchestra.run.smithi119.stdout:Total 347 kB/s | 255 kB 00:00 2024-04-22T14:59:00.880 INFO:teuthology.orchestra.run.smithi119.stdout:Running transaction check 2024-04-22T14:59:01.083 INFO:teuthology.orchestra.run.smithi119.stdout:Transaction check succeeded. 2024-04-22T14:59:01.083 INFO:teuthology.orchestra.run.smithi119.stdout:Running transaction test 2024-04-22T14:59:01.090 INFO:teuthology.orchestra.run.smithi119.stdout:Transaction test succeeded. 2024-04-22T14:59:01.090 INFO:teuthology.orchestra.run.smithi119.stdout:Running transaction 2024-04-22T14:59:01.220 INFO:teuthology.orchestra.run.smithi119.stdout: Preparing : 1/1 2024-04-22T14:59:01.299 INFO:teuthology.orchestra.run.smithi119.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T14:59:01.299 INFO:teuthology.orchestra.run.smithi119.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T14:59:01.984 INFO:teuthology.orchestra.run.smithi119.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T14:59:01.985 INFO:teuthology.orchestra.run.smithi119.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T14:59:02.701 INFO:teuthology.orchestra.run.smithi119.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T14:59:02.701 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T14:59:02.701 INFO:teuthology.orchestra.run.smithi119.stdout:Reinstalled: 2024-04-22T14:59:02.701 INFO:teuthology.orchestra.run.smithi119.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-22T14:59:02.701 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T14:59:02.701 INFO:teuthology.orchestra.run.smithi119.stdout:Complete! 2024-04-22T14:59:02.860 DEBUG:teuthology.orchestra.run.smithi119:> rpm -q kernel | sort -rV | head -n 1 2024-04-22T14:59:02.948 INFO:teuthology.orchestra.run.smithi119.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-22T14:59:02.948 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-22T14:59:02.949 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-22T14:59:02.949 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-04-22T14:59:02.949 DEBUG:teuthology.orchestra.run.smithi119:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-22T14:59:02.979 INFO:teuthology.orchestra.run.smithi119.stdout:ttyS1 2024-04-22T14:59:02.999 DEBUG:teuthology.parallel:result is None 2024-04-22T14:59:05.036 INFO:teuthology.orchestra.run.smithi186.stdout:lab-extras 27 kB/s | 1.7 kB 00:00 2024-04-22T14:59:06.892 INFO:teuthology.orchestra.run.smithi186.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-22T14:59:06.892 INFO:teuthology.orchestra.run.smithi186.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-22T14:59:06.985 INFO:teuthology.orchestra.run.smithi186.stdout:Dependencies resolved. 2024-04-22T14:59:06.986 INFO:teuthology.orchestra.run.smithi186.stdout:Nothing to do. 2024-04-22T14:59:06.986 INFO:teuthology.orchestra.run.smithi186.stdout:Complete! 2024-04-22T14:59:07.096 DEBUG:teuthology.orchestra.run.smithi186:> echo no | sudo yum reinstall kernel || true 2024-04-22T14:59:07.598 INFO:teuthology.orchestra.run.smithi186.stdout:Last metadata expiration check: 0:00:02 ago on Mon 22 Apr 2024 02:59:05 PM UTC. 2024-04-22T14:59:07.723 INFO:teuthology.orchestra.run.smithi186.stdout:Dependencies resolved. 2024-04-22T14:59:07.728 INFO:teuthology.orchestra.run.smithi186.stdout:================================================================================ 2024-04-22T14:59:07.728 INFO:teuthology.orchestra.run.smithi186.stdout: Package Architecture Version Repository Size 2024-04-22T14:59:07.728 INFO:teuthology.orchestra.run.smithi186.stdout:================================================================================ 2024-04-22T14:59:07.728 INFO:teuthology.orchestra.run.smithi186.stdout:Reinstalling: 2024-04-22T14:59:07.728 INFO:teuthology.orchestra.run.smithi186.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T14:59:07.728 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T14:59:07.728 INFO:teuthology.orchestra.run.smithi186.stdout:Transaction Summary 2024-04-22T14:59:07.728 INFO:teuthology.orchestra.run.smithi186.stdout:================================================================================ 2024-04-22T14:59:07.728 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T14:59:07.728 INFO:teuthology.orchestra.run.smithi186.stdout:Total download size: 255 k 2024-04-22T14:59:07.728 INFO:teuthology.orchestra.run.smithi186.stdout:Installed size: 0 2024-04-22T14:59:07.729 INFO:teuthology.orchestra.run.smithi186.stderr:Operation aborted. 2024-04-22T14:59:07.792 INFO:teuthology.orchestra.run.smithi186.stdout:Is this ok [y/N]: 2024-04-22T14:59:07.792 DEBUG:teuthology.orchestra.run.smithi186:> sudo yum reinstall -y kernel || true 2024-04-22T14:59:08.288 INFO:teuthology.orchestra.run.smithi186.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 02:59:05 PM UTC. 2024-04-22T14:59:08.404 INFO:teuthology.orchestra.run.smithi186.stdout:Dependencies resolved. 2024-04-22T14:59:08.405 INFO:teuthology.orchestra.run.smithi186.stdout:================================================================================ 2024-04-22T14:59:08.405 INFO:teuthology.orchestra.run.smithi186.stdout: Package Architecture Version Repository Size 2024-04-22T14:59:08.405 INFO:teuthology.orchestra.run.smithi186.stdout:================================================================================ 2024-04-22T14:59:08.405 INFO:teuthology.orchestra.run.smithi186.stdout:Reinstalling: 2024-04-22T14:59:08.405 INFO:teuthology.orchestra.run.smithi186.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T14:59:08.405 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T14:59:08.405 INFO:teuthology.orchestra.run.smithi186.stdout:Transaction Summary 2024-04-22T14:59:08.406 INFO:teuthology.orchestra.run.smithi186.stdout:================================================================================ 2024-04-22T14:59:08.406 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T14:59:08.406 INFO:teuthology.orchestra.run.smithi186.stdout:Total download size: 255 k 2024-04-22T14:59:08.406 INFO:teuthology.orchestra.run.smithi186.stdout:Installed size: 0 2024-04-22T14:59:08.406 INFO:teuthology.orchestra.run.smithi186.stdout:Downloading Packages: 2024-04-22T14:59:08.563 INFO:teuthology.orchestra.run.smithi186.stdout:kernel-5.14.0-437.el9.x86_64.rpm 2.8 MB/s | 255 kB 00:00 2024-04-22T14:59:08.563 INFO:teuthology.orchestra.run.smithi186.stdout:-------------------------------------------------------------------------------- 2024-04-22T14:59:08.563 INFO:teuthology.orchestra.run.smithi186.stdout:Total 1.6 MB/s | 255 kB 00:00 2024-04-22T14:59:08.582 INFO:teuthology.orchestra.run.smithi186.stdout:Running transaction check 2024-04-22T14:59:08.781 INFO:teuthology.orchestra.run.smithi186.stdout:Transaction check succeeded. 2024-04-22T14:59:08.781 INFO:teuthology.orchestra.run.smithi186.stdout:Running transaction test 2024-04-22T14:59:08.786 INFO:teuthology.orchestra.run.smithi186.stdout:Transaction test succeeded. 2024-04-22T14:59:08.786 INFO:teuthology.orchestra.run.smithi186.stdout:Running transaction 2024-04-22T14:59:08.915 INFO:teuthology.orchestra.run.smithi186.stdout: Preparing : 1/1 2024-04-22T14:59:09.006 INFO:teuthology.orchestra.run.smithi186.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T14:59:09.006 INFO:teuthology.orchestra.run.smithi186.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T14:59:09.646 INFO:teuthology.orchestra.run.smithi186.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T14:59:09.646 INFO:teuthology.orchestra.run.smithi186.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T14:59:10.332 INFO:teuthology.orchestra.run.smithi186.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T14:59:10.332 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T14:59:10.332 INFO:teuthology.orchestra.run.smithi186.stdout:Reinstalled: 2024-04-22T14:59:10.332 INFO:teuthology.orchestra.run.smithi186.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-22T14:59:10.333 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T14:59:10.333 INFO:teuthology.orchestra.run.smithi186.stdout:Complete! 2024-04-22T14:59:10.543 DEBUG:teuthology.orchestra.run.smithi186:> rpm -q kernel | sort -rV | head -n 1 2024-04-22T14:59:10.589 INFO:teuthology.orchestra.run.smithi186.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-22T14:59:10.590 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-22T14:59:10.590 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-22T14:59:10.590 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-04-22T14:59:10.590 DEBUG:teuthology.orchestra.run.smithi186:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-22T14:59:10.623 INFO:teuthology.orchestra.run.smithi186.stdout:ttyS1 2024-04-22T14:59:10.645 DEBUG:teuthology.parallel:result is None 2024-04-22T14:59:10.645 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-22T14:59:10.653 INFO:teuthology.task.internal:Creating test directory... 2024-04-22T14:59:10.654 DEBUG:teuthology.orchestra.run.smithi119:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-22T14:59:10.656 DEBUG:teuthology.orchestra.run.smithi186:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-22T14:59:10.703 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-22T14:59:10.709 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-22T14:59:10.716 INFO:teuthology.task.internal:Creating archive directory... 2024-04-22T14:59:10.716 DEBUG:teuthology.orchestra.run.smithi119:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-22T14:59:10.719 DEBUG:teuthology.orchestra.run.smithi186:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-22T14:59:10.790 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-22T14:59:10.797 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-22T14:59:10.797 DEBUG:teuthology.orchestra.run.smithi119:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-22T14:59:10.799 DEBUG:teuthology.orchestra.run.smithi186:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-22T14:59:10.834 INFO:teuthology.orchestra.run.smithi119.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T14:59:10.840 INFO:teuthology.orchestra.run.smithi186.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T14:59:10.844 INFO:teuthology.orchestra.run.smithi119.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T14:59:10.849 INFO:teuthology.orchestra.run.smithi186.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T14:59:10.851 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-22T14:59:10.859 INFO:teuthology.task.internal:Configuring sudo... 2024-04-22T14:59:10.860 DEBUG:teuthology.orchestra.run.smithi119:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-22T14:59:10.889 DEBUG:teuthology.orchestra.run.smithi186:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-22T14:59:10.915 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-22T14:59:10.924 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-22T14:59:10.925 DEBUG:teuthology.orchestra.run.smithi119:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-22T14:59:10.958 DEBUG:teuthology.orchestra.run.smithi186:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-22T14:59:10.973 DEBUG:teuthology.orchestra.run.smithi119:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T14:59:11.057 DEBUG:teuthology.orchestra.run.smithi119:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T14:59:11.104 DEBUG:teuthology.orchestra.run.smithi119:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T14:59:11.186 DEBUG:teuthology.orchestra.run.smithi119:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T14:59:11.228 DEBUG:teuthology.orchestra.run.smithi119:> set -ex 2024-04-22T14:59:11.228 DEBUG:teuthology.orchestra.run.smithi119:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-22T14:59:11.292 DEBUG:teuthology.orchestra.run.smithi186:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T14:59:11.330 DEBUG:teuthology.orchestra.run.smithi186:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T14:59:11.392 DEBUG:teuthology.orchestra.run.smithi186:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T14:59:11.473 DEBUG:teuthology.orchestra.run.smithi186:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T14:59:11.512 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-04-22T14:59:11.512 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-22T14:59:11.575 DEBUG:teuthology.orchestra.run.smithi119:> sudo service rsyslog restart 2024-04-22T14:59:11.577 DEBUG:teuthology.orchestra.run.smithi186:> sudo service rsyslog restart 2024-04-22T14:59:11.612 INFO:teuthology.orchestra.run.smithi119.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T14:59:11.652 INFO:teuthology.orchestra.run.smithi186.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T14:59:12.058 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-22T14:59:12.066 INFO:teuthology.task.internal:Starting timer... 2024-04-22T14:59:12.066 INFO:teuthology.run_tasks:Running task pcp... 2024-04-22T14:59:12.076 INFO:teuthology.run_tasks:Running task selinux... 2024-04-22T14:59:12.084 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-04-22T14:59:12.084 DEBUG:teuthology.orchestra.run.smithi119:> sudo service auditd rotate 2024-04-22T14:59:12.142 INFO:teuthology.orchestra.run.smithi119.stdout:Rotating logs: 2024-04-22T14:59:12.144 DEBUG:teuthology.orchestra.run.smithi186:> sudo service auditd rotate 2024-04-22T14:59:12.204 INFO:teuthology.orchestra.run.smithi186.stdout:Rotating logs: 2024-04-22T14:59:12.205 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-22T14:59:12.205 DEBUG:teuthology.orchestra.run.smithi119:> /usr/sbin/getenforce 2024-04-22T14:59:12.227 INFO:teuthology.orchestra.run.smithi119.stdout:Permissive 2024-04-22T14:59:12.228 DEBUG:teuthology.orchestra.run.smithi186:> /usr/sbin/getenforce 2024-04-22T14:59:12.256 INFO:teuthology.orchestra.run.smithi186.stdout:Permissive 2024-04-22T14:59:12.257 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi119.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi186.front.sepia.ceph.com': 'permissive'} 2024-04-22T14:59:12.257 DEBUG:teuthology.orchestra.run.smithi119:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-22T14:59:12.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T14:59:12.294 DEBUG:teuthology.orchestra.run.smithi186:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-22T14:59:12.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T14:59:12.321 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-22T14:59:12.321 DEBUG:teuthology.orchestra.run.smithi119:> sudo /usr/sbin/setenforce permissive 2024-04-22T14:59:12.358 DEBUG:teuthology.orchestra.run.smithi186:> sudo /usr/sbin/setenforce permissive 2024-04-22T14:59:12.386 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-22T14:59:12.397 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-22T14:59:12.414 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-22T14:59:12.415 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi119.front.sepia.ceph.com,smithi186.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-22T15:03:18.653 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi119.front.sepia.ceph.com'), Remote(name='ubuntu@smithi186.front.sepia.ceph.com')] 2024-04-22T15:03:18.654 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi119.front.sepia.ceph.com' 2024-04-22T15:03:18.655 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi119.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T15:03:18.747 DEBUG:teuthology.orchestra.run.smithi119:> true 2024-04-22T15:03:18.826 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi119.front.sepia.ceph.com' 2024-04-22T15:03:18.826 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi186.front.sepia.ceph.com' 2024-04-22T15:03:18.827 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi186.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T15:03:18.954 DEBUG:teuthology.orchestra.run.smithi186:> true 2024-04-22T15:03:19.044 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi186.front.sepia.ceph.com' 2024-04-22T15:03:19.044 INFO:teuthology.run_tasks:Running task clock... 2024-04-22T15:03:19.077 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-22T15:03:19.078 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-22T15:03:19.078 DEBUG:teuthology.orchestra.run.smithi119:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T15:03:19.090 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-22T15:03:19.090 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T15:03:19.112 INFO:teuthology.orchestra.run.smithi119.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-22T15:03:19.122 INFO:teuthology.orchestra.run.smithi186.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-22T15:03:19.126 INFO:teuthology.orchestra.run.smithi119.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-22T15:03:19.137 INFO:teuthology.orchestra.run.smithi186.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-22T15:03:19.150 INFO:teuthology.orchestra.run.smithi119.stderr:sudo: ntpd: command not found 2024-04-22T15:03:19.161 INFO:teuthology.orchestra.run.smithi119.stdout:506 Cannot talk to daemon 2024-04-22T15:03:19.161 INFO:teuthology.orchestra.run.smithi186.stderr:sudo: ntpd: command not found 2024-04-22T15:03:19.172 INFO:teuthology.orchestra.run.smithi186.stdout:506 Cannot talk to daemon 2024-04-22T15:03:19.175 INFO:teuthology.orchestra.run.smithi119.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-22T15:03:19.186 INFO:teuthology.orchestra.run.smithi186.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-22T15:03:19.187 INFO:teuthology.orchestra.run.smithi119.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-22T15:03:19.200 INFO:teuthology.orchestra.run.smithi186.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-22T15:03:19.240 INFO:teuthology.orchestra.run.smithi119.stderr:bash: line 1: ntpq: command not found 2024-04-22T15:03:19.243 INFO:teuthology.orchestra.run.smithi119.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T15:03:19.243 INFO:teuthology.orchestra.run.smithi119.stdout:=============================================================================== 2024-04-22T15:03:19.243 INFO:teuthology.orchestra.run.smithi119.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:19.243 INFO:teuthology.orchestra.run.smithi119.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:19.244 INFO:teuthology.orchestra.run.smithi119.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:19.244 INFO:teuthology.orchestra.run.smithi119.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:19.254 INFO:teuthology.orchestra.run.smithi186.stderr:bash: line 1: ntpq: command not found 2024-04-22T15:03:19.256 INFO:teuthology.orchestra.run.smithi186.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T15:03:19.256 INFO:teuthology.orchestra.run.smithi186.stdout:=============================================================================== 2024-04-22T15:03:19.256 INFO:teuthology.orchestra.run.smithi186.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:19.257 INFO:teuthology.orchestra.run.smithi186.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:19.257 INFO:teuthology.orchestra.run.smithi186.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:19.257 INFO:teuthology.orchestra.run.smithi186.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:19.258 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-22T15:03:19.445 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '5883d494d3973b30b3f03e4a3f0e1e06e3c131aa'} 2024-04-22T15:03:19.445 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-22T15:03:19.447 INFO:tasks.cephadm:Cluster fsid is 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:03:19.447 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-22T15:03:19.447 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-04-22T15:03:19.447 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi119': '172.21.15.119', 'mon.smithi186': '172.21.15.186'} 2024-04-22T15:03:19.447 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-22T15:03:19.448 DEBUG:teuthology.orchestra.run.smithi119:> sudo hostname $(hostname -s) 2024-04-22T15:03:19.473 DEBUG:teuthology.orchestra.run.smithi186:> sudo hostname $(hostname -s) 2024-04-22T15:03:19.499 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-22T15:03:19.500 DEBUG:teuthology.orchestra.run.smithi119:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-22T15:03:19.807 INFO:teuthology.orchestra.run.smithi119.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 22 15:03 /home/ubuntu/cephtest/cephadm 2024-04-22T15:03:19.808 DEBUG:teuthology.orchestra.run.smithi186:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-22T15:03:19.980 INFO:teuthology.orchestra.run.smithi186.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 22 15:03 /home/ubuntu/cephtest/cephadm 2024-04-22T15:03:19.980 DEBUG:teuthology.orchestra.run.smithi119:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-22T15:03:20.003 DEBUG:teuthology.orchestra.run.smithi186:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-22T15:03:20.029 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-22T15:03:20.029 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-22T15:03:20.047 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-22T15:03:21.456 INFO:teuthology.orchestra.run.smithi186.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T15:03:21.493 INFO:teuthology.orchestra.run.smithi119.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T15:03:44.019 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:03:44.019 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-22T15:03:44.019 INFO:teuthology.orchestra.run.smithi119.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-22T15:03:44.019 INFO:teuthology.orchestra.run.smithi119.stdout: "repo_digests": [ 2024-04-22T15:03:44.019 INFO:teuthology.orchestra.run.smithi119.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-22T15:03:44.020 INFO:teuthology.orchestra.run.smithi119.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-22T15:03:44.020 INFO:teuthology.orchestra.run.smithi119.stdout: ] 2024-04-22T15:03:44.020 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:03:44.276 INFO:teuthology.orchestra.run.smithi186.stdout:{ 2024-04-22T15:03:44.276 INFO:teuthology.orchestra.run.smithi186.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-22T15:03:44.276 INFO:teuthology.orchestra.run.smithi186.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-22T15:03:44.276 INFO:teuthology.orchestra.run.smithi186.stdout: "repo_digests": [ 2024-04-22T15:03:44.276 INFO:teuthology.orchestra.run.smithi186.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-22T15:03:44.276 INFO:teuthology.orchestra.run.smithi186.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-22T15:03:44.276 INFO:teuthology.orchestra.run.smithi186.stdout: ] 2024-04-22T15:03:44.276 INFO:teuthology.orchestra.run.smithi186.stdout:} 2024-04-22T15:03:44.295 DEBUG:teuthology.orchestra.run.smithi119:> sudo mkdir -p /etc/ceph 2024-04-22T15:03:44.322 DEBUG:teuthology.orchestra.run.smithi186:> sudo mkdir -p /etc/ceph 2024-04-22T15:03:44.348 DEBUG:teuthology.orchestra.run.smithi119:> sudo chmod 777 /etc/ceph 2024-04-22T15:03:44.388 DEBUG:teuthology.orchestra.run.smithi186:> sudo chmod 777 /etc/ceph 2024-04-22T15:03:44.412 INFO:tasks.cephadm:Writing seed config... 2024-04-22T15:03:44.413 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-22T15:03:44.413 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-22T15:03:44.413 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-22T15:03:44.413 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-22T15:03:44.413 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-22T15:03:44.413 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-22T15:03:44.413 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-22T15:03:44.413 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-22T15:03:44.414 DEBUG:teuthology.orchestra.run.smithi119:> set -ex 2024-04-22T15:03:44.414 DEBUG:teuthology.orchestra.run.smithi119:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-22T15:03:44.444 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 74739db6-00b9-11ef-bc93-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-22T15:03:44.444 DEBUG:teuthology.orchestra.run.smithi119:mon.smithi119> sudo journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi119.service 2024-04-22T15:03:44.494 INFO:tasks.cephadm:Bootstrapping... 2024-04-22T15:03:44.494 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 74739db6-00b9-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.119 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:03:44.685 INFO:teuthology.orchestra.run.smithi119.stderr:-------------------------------------------------------------------------------- 2024-04-22T15:03:44.685 INFO:teuthology.orchestra.run.smithi119.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '74739db6-00b9-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.119', '--skip-admin-label'] 2024-04-22T15:03:44.708 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: 4.9.4-dev 2024-04-22T15:03:44.710 INFO:teuthology.orchestra.run.smithi119.stderr:Verifying podman|docker is present... 2024-04-22T15:03:44.732 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: 4.9.4-dev 2024-04-22T15:03:44.735 INFO:teuthology.orchestra.run.smithi119.stderr:Verifying lvm2 is present... 2024-04-22T15:03:44.735 INFO:teuthology.orchestra.run.smithi119.stderr:Verifying time synchronization is in place... 2024-04-22T15:03:44.740 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-22T15:03:44.747 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: inactive 2024-04-22T15:03:44.753 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: enabled 2024-04-22T15:03:44.760 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: active 2024-04-22T15:03:44.760 INFO:teuthology.orchestra.run.smithi119.stderr:Unit chronyd.service is enabled and running 2024-04-22T15:03:44.760 INFO:teuthology.orchestra.run.smithi119.stderr:Repeating the final host check... 2024-04-22T15:03:44.779 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: 4.9.4-dev 2024-04-22T15:03:44.782 INFO:teuthology.orchestra.run.smithi119.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-22T15:03:44.782 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl is present 2024-04-22T15:03:44.783 INFO:teuthology.orchestra.run.smithi119.stderr:lvcreate is present 2024-04-22T15:03:44.790 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-22T15:03:44.793 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: inactive 2024-04-22T15:03:44.800 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: enabled 2024-04-22T15:03:44.808 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: active 2024-04-22T15:03:44.808 INFO:teuthology.orchestra.run.smithi119.stderr:Unit chronyd.service is enabled and running 2024-04-22T15:03:44.808 INFO:teuthology.orchestra.run.smithi119.stderr:Host looks OK 2024-04-22T15:03:44.808 INFO:teuthology.orchestra.run.smithi119.stderr:Cluster fsid: 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:03:44.808 INFO:teuthology.orchestra.run.smithi119.stderr:Acquiring lock 140052034413856 on /run/cephadm/74739db6-00b9-11ef-bc93-c7b262605968.lock 2024-04-22T15:03:44.808 INFO:teuthology.orchestra.run.smithi119.stderr:Lock 140052034413856 acquired on /run/cephadm/74739db6-00b9-11ef-bc93-c7b262605968.lock 2024-04-22T15:03:44.808 INFO:teuthology.orchestra.run.smithi119.stderr:Verifying IP 172.21.15.119 port 3300 ... 2024-04-22T15:03:44.808 INFO:teuthology.orchestra.run.smithi119.stderr:Verifying IP 172.21.15.119 port 6789 ... 2024-04-22T15:03:44.808 INFO:teuthology.orchestra.run.smithi119.stderr:Base mon IP is 172.21.15.119, final addrv is [v2:172.21.15.119:3300,v1:172.21.15.119:6789] 2024-04-22T15:03:44.810 INFO:teuthology.orchestra.run.smithi119.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.119 metric 100 2024-04-22T15:03:44.810 INFO:teuthology.orchestra.run.smithi119.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.119 metric 100 2024-04-22T15:03:44.813 INFO:teuthology.orchestra.run.smithi119.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-22T15:03:44.813 INFO:teuthology.orchestra.run.smithi119.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-04-22T15:03:44.813 INFO:teuthology.orchestra.run.smithi119.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-04-22T15:03:44.815 INFO:teuthology.orchestra.run.smithi119.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-22T15:03:44.815 INFO:teuthology.orchestra.run.smithi119.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-22T15:03:44.815 INFO:teuthology.orchestra.run.smithi119.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-22T15:03:44.815 INFO:teuthology.orchestra.run.smithi119.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-22T15:03:44.815 INFO:teuthology.orchestra.run.smithi119.stderr:/sbin/ip: inet6 fe80::aab7:c035:2b33:4570/64 scope link noprefixroute 2024-04-22T15:03:44.815 INFO:teuthology.orchestra.run.smithi119.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-22T15:03:44.816 INFO:teuthology.orchestra.run.smithi119.stderr:Mon IP `172.21.15.119` is in CIDR network `172.21.0.0/20` 2024-04-22T15:03:44.816 INFO:teuthology.orchestra.run.smithi119.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-22T15:03:44.817 INFO:teuthology.orchestra.run.smithi119.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T15:03:44.929 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-22T15:03:45.348 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: Getting image source signatures 2024-04-22T15:03:45.348 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-22T15:03:45.348 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-22T15:03:45.349 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-22T15:03:45.349 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-22T15:03:45.349 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-22T15:03:45.349 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-22T15:03:45.506 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: Writing manifest to image destination 2024-04-22T15:03:45.658 INFO:teuthology.orchestra.run.smithi119.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-22T15:03:46.086 INFO:teuthology.orchestra.run.smithi119.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-22T15:03:46.666 INFO:teuthology.orchestra.run.smithi119.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-22T15:03:46.666 INFO:teuthology.orchestra.run.smithi119.stderr:Extracting ceph user uid/gid from container image... 2024-04-22T15:03:47.078 INFO:teuthology.orchestra.run.smithi119.stderr:stat: 167 167 2024-04-22T15:03:47.563 INFO:teuthology.orchestra.run.smithi119.stderr:Creating initial keys... 2024-04-22T15:03:47.946 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph-authtool: AQBTfCZmix4xMxAAKHXtvpaeW6HTFWMQCYad4w== 2024-04-22T15:03:48.851 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph-authtool: AQBUfCZmSe+KLRAAuXND81G65nYZ+zwSNWcGMQ== 2024-04-22T15:03:49.682 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph-authtool: AQBVfCZmoGPqJRAAkjTscEBAgxc/b1FIbDp3oA== 2024-04-22T15:03:50.164 INFO:teuthology.orchestra.run.smithi119.stderr:Creating initial monmap... 2024-04-22T15:03:50.589 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-22T15:03:50.589 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-22T15:03:50.589 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:03:50.589 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-22T15:03:51.070 INFO:teuthology.orchestra.run.smithi119.stderr:monmaptool for smithi119 [v2:172.21.15.119:3300,v1:172.21.15.119:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-22T15:03:51.070 INFO:teuthology.orchestra.run.smithi119.stderr:setting min_mon_release = octopus 2024-04-22T15:03:51.070 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/monmaptool: set fsid to 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:03:51.070 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-22T15:03:51.070 INFO:teuthology.orchestra.run.smithi119.stderr: 2024-04-22T15:03:51.070 INFO:teuthology.orchestra.run.smithi119.stderr:Creating mon... 2024-04-22T15:03:51.983 INFO:teuthology.orchestra.run.smithi119.stderr:create mon.smithi119 on 2024-04-22T15:03:52.184 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-22T15:03:52.333 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-74739db6-00b9-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-74739db6-00b9-11ef-bc93-c7b262605968.target. 2024-04-22T15:03:52.334 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-74739db6-00b9-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-74739db6-00b9-11ef-bc93-c7b262605968.target. 2024-04-22T15:03:52.884 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: Failed to reset failed state of unit ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi119.service: Unit ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi119.service not loaded. 2024-04-22T15:03:52.920 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: Created symlink /etc/systemd/system/ceph-74739db6-00b9-11ef-bc93-c7b262605968.target.wants/ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi119.service → /etc/systemd/system/ceph-74739db6-00b9-11ef-bc93-c7b262605968@.service. 2024-04-22T15:03:56.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:03:56 smithi119 ceph-mon[21490]: mon.smithi119 is new leader, mons smithi119 in quorum (ranks 0) 2024-04-22T15:03:57.117 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: disabled 2024-04-22T15:03:57.124 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: inactive 2024-04-22T15:03:57.125 INFO:teuthology.orchestra.run.smithi119.stderr:firewalld.service is not enabled 2024-04-22T15:03:57.125 INFO:teuthology.orchestra.run.smithi119.stderr:Not possible to enable service . firewalld.service is not available 2024-04-22T15:03:57.126 INFO:teuthology.orchestra.run.smithi119.stderr:Waiting for mon to start... 2024-04-22T15:03:57.126 INFO:teuthology.orchestra.run.smithi119.stderr:Waiting for mon... 2024-04-22T15:03:57.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:03:57 smithi119 bash[21461]: 2fd515f919567dbe06b77a85615af950322efce2f463fe16145eca3f49eb1061 2024-04-22T15:03:57.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:03:57 smithi119 systemd[1]: Started Ceph mon.smithi119 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:03:57.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:03:57 smithi119 ceph-mon[21490]: mon.smithi119 is new leader, mons smithi119 in quorum (ranks 0) 2024-04-22T15:03:57.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:03:57 smithi119 ceph-mon[21490]: monmap e1: 1 mons at {smithi119=[v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0]} 2024-04-22T15:03:57.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:03:57 smithi119 ceph-mon[21490]: fsmap 2024-04-22T15:03:57.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:03:57 smithi119 ceph-mon[21490]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T15:03:57.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:03:57 smithi119 ceph-mon[21490]: mgrmap e1: no daemons active 2024-04-22T15:03:59.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:03:59 smithi119 ceph-mon[21490]: from='client.? 172.21.15.119:0/2845015931' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:04:00.680 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: cluster: 2024-04-22T15:04:00.680 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: id: 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:00.680 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-22T15:04:00.680 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 2024-04-22T15:04:00.680 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: services: 2024-04-22T15:04:00.681 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi119 (age 3s) 2024-04-22T15:04:00.681 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-22T15:04:00.681 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-22T15:04:00.681 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 2024-04-22T15:04:00.681 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: data: 2024-04-22T15:04:00.681 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-22T15:04:00.681 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-22T15:04:00.681 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-22T15:04:00.681 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: pgs: 2024-04-22T15:04:00.681 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 2024-04-22T15:04:03.155 INFO:teuthology.orchestra.run.smithi119.stderr:mon is available 2024-04-22T15:04:03.155 INFO:teuthology.orchestra.run.smithi119.stderr:Assimilating anything we can from ceph.conf... 2024-04-22T15:04:03.593 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 2024-04-22T15:04:03.593 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: [global] 2024-04-22T15:04:03.593 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: fsid = 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:03.594 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.119:3300,v1:172.21.15.119:6789] 2024-04-22T15:04:03.594 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-22T15:04:03.594 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-22T15:04:03.594 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-22T15:04:03.594 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-22T15:04:03.594 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 2024-04-22T15:04:03.594 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: [mgr] 2024-04-22T15:04:03.594 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-22T15:04:03.594 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 2024-04-22T15:04:03.594 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: [osd] 2024-04-22T15:04:03.594 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-22T15:04:03.594 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-22T15:04:03.848 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:03 smithi119 ceph-mon[21490]: from='client.? 172.21.15.119:0/2748679424' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T15:04:03.849 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:03 smithi119 ceph-mon[21490]: from='client.? 172.21.15.119:0/2748679424' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-22T15:04:04.118 INFO:teuthology.orchestra.run.smithi119.stderr:Generating new minimal ceph.conf... 2024-04-22T15:04:04.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:04 smithi119 ceph-mon[21490]: from='client.? 172.21.15.119:0/2987525411' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:04:05.108 INFO:teuthology.orchestra.run.smithi119.stderr:Restarting the monitor... 2024-04-22T15:04:05.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:05 smithi119 systemd[1]: Stopping Ceph mon.smithi119 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:04:05.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:05 smithi119 bash[21710]: Error: no container with name or ID "ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon.smithi119" found: no such container 2024-04-22T15:04:05.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:05 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119[21486]: 2024-04-22T15:04:05.263+0000 7fb0cfea4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi119 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:04:05.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:05 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119[21486]: 2024-04-22T15:04:05.263+0000 7fb0cfea4700 -1 mon.smithi119@0(leader) e1 *** Got Signal Terminated *** 2024-04-22T15:04:05.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:05 smithi119 podman[21720]: 2024-04-22 15:04:05.37656978 +0000 UTC m=+0.214253103 container died 2fd515f919567dbe06b77a85615af950322efce2f463fe16145eca3f49eb1061 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, GIT_CLEAN=True, version=8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vcs-type=git, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T15:04:05.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:05 smithi119 podman[21720]: 2024-04-22 15:04:05.730314401 +0000 UTC m=+0.567997732 container cleanup 2fd515f919567dbe06b77a85615af950322efce2f463fe16145eca3f49eb1061 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, distribution-scope=public, release=754, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, version=8, vcs-type=git) 2024-04-22T15:04:05.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:05 smithi119 bash[21720]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119 2024-04-22T15:04:05.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:05 smithi119 bash[21746]: Error: no container with name or ID "ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon.smithi119" found: no such container 2024-04-22T15:04:06.340 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 podman[21734]: 2024-04-22 15:04:06.006075858 +0000 UTC m=+0.634238305 container remove 2fd515f919567dbe06b77a85615af950322efce2f463fe16145eca3f49eb1061 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, name=centos-stream, vendor=Red Hat, Inc., version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-22T15:04:06.340 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi119.service: Deactivated successfully. 2024-04-22T15:04:06.341 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 systemd[1]: Stopped Ceph mon.smithi119 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:04:06.341 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 systemd[1]: Starting Ceph mon.smithi119 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:04:06.341 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 podman[21816]: 2024-04-22 15:04:06.231911181 +0000 UTC m=+0.112118509 container create e31b6591d879b56cac373b321e58fad7bc689660f99a38872f480090f3e06685 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., ceph=True, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, version=8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.buildah.version=1.19.8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-22T15:04:06.341 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 podman[21816]: 2024-04-22 15:04:06.137801305 +0000 UTC m=+0.018008602 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-22T15:04:06.503 INFO:teuthology.orchestra.run.smithi119.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-22T15:04:06.624 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 podman[21816]: 2024-04-22 15:04:06.40076434 +0000 UTC m=+0.280971623 container init e31b6591d879b56cac373b321e58fad7bc689660f99a38872f480090f3e06685 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, vcs-type=git, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, architecture=x86_64, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream) 2024-04-22T15:04:06.624 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 podman[21816]: 2024-04-22 15:04:06.403742696 +0000 UTC m=+0.283949980 container start e31b6591d879b56cac373b321e58fad7bc689660f99a38872f480090f3e06685 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, vcs-type=git, distribution-scope=public, release=754, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, ceph=True, version=8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-22T15:04:06.624 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T15:04:06.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-22T15:04:06.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: pidfile_write: ignore empty --pid-file 2024-04-22T15:04:06.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: load: jerasure load: lrc 2024-04-22T15:04:06.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: RocksDB version: 6.15.5 2024-04-22T15:04:06.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-22T15:04:06.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Compile date Apr 18 2022 2024-04-22T15:04:06.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: DB SUMMARY 2024-04-22T15:04:06.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: DB Session ID: 4RIUQN1DPQTJF4O97CEI 2024-04-22T15:04:06.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: CURRENT file: CURRENT 2024-04-22T15:04:06.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T15:04:06.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-22T15:04:06.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi119/store.db dir, Total Num: 1, files: 000008.sst 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi119/store.db: 000010.log size: 83537 ; 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.error_if_exists: 0 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.create_if_missing: 0 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.paranoid_checks: 1 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.env: 0x561cf0ac6860 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.fs: Posix File System 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.info_log: 0x561cf1cd1fa0 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.statistics: (nil) 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.use_fsync: 0 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_log_file_size: 0 2024-04-22T15:04:06.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.allow_fallocate: 1 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.use_direct_reads: 0 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.db_log_dir: 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi119/store.db 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T15:04:06.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T15:04:06.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T15:04:06.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T15:04:06.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T15:04:06.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T15:04:06.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T15:04:06.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T15:04:06.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.write_buffer_manager: 0x561cf1dc22d0 2024-04-22T15:04:06.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T15:04:06.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-22T15:04:06.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T15:04:06.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T15:04:06.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.unordered_write: 0 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.row_cache: None 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.wal_filter: None 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T15:04:06.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T15:04:06.630 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.preserve_deletes: 0 2024-04-22T15:04:06.630 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.two_write_queues: 0 2024-04-22T15:04:06.630 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T15:04:06.630 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.atomic_flush: 0 2024-04-22T15:04:06.630 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T15:04:06.630 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T15:04:06.630 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T15:04:06.630 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.log_readahead_size: 0 2024-04-22T15:04:06.630 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T15:04:06.630 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T15:04:06.630 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T15:04:06.630 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_background_jobs: 2 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_background_compactions: -1 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_subcompactions: 1 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T15:04:06.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T15:04:06.632 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_open_files: -1 2024-04-22T15:04:06.632 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T15:04:06.632 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T15:04:06.632 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T15:04:06.632 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T15:04:06.632 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_background_flushes: -1 2024-04-22T15:04:06.632 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Compression algorithms supported: 2024-04-22T15:04:06.632 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T15:04:06.633 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: kZSTD supported: 0 2024-04-22T15:04:06.633 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: kXpressCompression supported: 0 2024-04-22T15:04:06.633 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T15:04:06.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: kLZ4Compression supported: 1 2024-04-22T15:04:06.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: kBZip2Compression supported: 0 2024-04-22T15:04:06.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: kZlibCompression supported: 1 2024-04-22T15:04:06.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: kSnappyCompression supported: 1 2024-04-22T15:04:06.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T15:04:06.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi119/store.db/MANIFEST-000009 2024-04-22T15:04:06.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-22T15:04:06.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T15:04:06.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.merge_operator: 2024-04-22T15:04:06.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_filter: None 2024-04-22T15:04:06.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T15:04:06.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T15:04:06.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T15:04:06.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T15:04:06.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561cf1c9dd00) 2024-04-22T15:04:06.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: cache_index_and_filter_blocks: 1 2024-04-22T15:04:06.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T15:04:06.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T15:04:06.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: pin_top_level_index_and_filter: 1 2024-04-22T15:04:06.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: index_type: 0 2024-04-22T15:04:06.636 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: data_block_index_type: 0 2024-04-22T15:04:06.636 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: index_shortening: 1 2024-04-22T15:04:06.636 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T15:04:06.636 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: hash_index_allow_collision: 1 2024-04-22T15:04:06.636 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: checksum: 1 2024-04-22T15:04:06.636 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: no_block_cache: 0 2024-04-22T15:04:06.636 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_cache: 0x561cf1d08170 2024-04-22T15:04:06.636 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_cache_name: BinnedLRUCache 2024-04-22T15:04:06.636 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_cache_options: 2024-04-22T15:04:06.636 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: capacity : 536870912 2024-04-22T15:04:06.636 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: num_shard_bits : 4 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: strict_capacity_limit : 0 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: high_pri_pool_ratio: 0.000 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_cache_compressed: (nil) 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: persistent_cache: (nil) 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_size: 4096 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_size_deviation: 10 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_restart_interval: 16 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: index_block_restart_interval: 1 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: metadata_block_size: 4096 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: partition_filters: 0 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: use_delta_encoding: 1 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-22T15:04:06.637 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: whole_key_filtering: 1 2024-04-22T15:04:06.638 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: verify_compression: 0 2024-04-22T15:04:06.638 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: read_amp_bytes_per_bit: 0 2024-04-22T15:04:06.638 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: format_version: 4 2024-04-22T15:04:06.638 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: enable_index_compression: 1 2024-04-22T15:04:06.638 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_align: 0 2024-04-22T15:04:06.638 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T15:04:06.638 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T15:04:06.638 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compression: NoCompression 2024-04-22T15:04:06.638 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T15:04:06.638 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T15:04:06.638 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T15:04:06.638 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.num_levels: 7 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T15:04:06.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T15:04:06.640 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T15:04:06.640 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:04:06.640 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T15:04:06.640 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T15:04:06.641 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T15:04:06.641 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T15:04:06.641 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T15:04:06.641 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T15:04:06.641 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T15:04:06.641 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T15:04:06.642 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T15:04:06.642 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T15:04:06.642 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T15:04:06.642 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T15:04:06.655 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T15:04:06.656 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T15:04:06.656 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T15:04:06.656 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.arena_block_size: 4194304 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T15:04:06.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T15:04:06.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T15:04:06.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T15:04:06.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T15:04:06.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T15:04:06.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T15:04:06.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.table_properties_collectors: 2024-04-22T15:04:06.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.inplace_update_support: 0 2024-04-22T15:04:06.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T15:04:06.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T15:04:06.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T15:04:06.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.bloom_locality: 0 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.max_successive_merges: 0 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.ttl: 2592000 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.enable_blob_files: false 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.min_blob_size: 0 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T15:04:06.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T15:04:06.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T15:04:06.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi119/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T15:04:06.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-22T15:04:06.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-22T15:04:06.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798246429237, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-22T15:04:06.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-22T15:04:06.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-22T15:04:06.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798246429930, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79865, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78110, "index_size": 243, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9930, "raw_average_key_size": 46, "raw_value_size": 72361, "raw_average_value_size": 341, "num_data_blocks": 11, "num_entries": 212, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713798246, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "8f6587a7-0272-4f02-ac4a-ea1b6873a482", "db_session_id": "4RIUQN1DPQTJF4O97CEI"}} 2024-04-22T15:04:06.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-22T15:04:06.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798246430248, "job": 1, "event": "recovery_finished"} 2024-04-22T15:04:06.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi119/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T15:04:06.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x561cf1ceea80 2024-04-22T15:04:06.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: DB pointer 0x561cf1cfe000 2024-04-22T15:04:06.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-22T15:04:06.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-22T15:04:06.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ** DB Stats ** 2024-04-22T15:04:06.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:04:06.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T15:04:06.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:04:06.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:04:06.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T15:04:06.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-22T15:04:06.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:04:06.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:04:06.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ** Compaction Stats [default] ** 2024-04-22T15:04:06.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:04:06.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:04:06.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: L0 2/0 79.79 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:06.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Sum 2/0 79.79 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:06.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:06.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:04:06.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ** Compaction Stats [default] ** 2024-04-22T15:04:06.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:04:06.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:04:06.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:06.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:04:06.663 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T15:04:06.663 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:04:06.663 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:04:06.669 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:04:06.669 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:04:06.669 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Cumulative compaction: 0.00 GB write, 27.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:04:06.669 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Interval compaction: 0.00 GB write, 27.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:04:06.669 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:04:06.669 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:04:06.669 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:04:06.669 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:04:06.669 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ** Compaction Stats [default] ** 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: L0 2/0 79.79 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Sum 2/0 79.79 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ** Compaction Stats [default] ** 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T15:04:06.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:04:06.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:04:06.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:04:06.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:04:06.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Cumulative compaction: 0.00 GB write, 26.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:04:06.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:04:06.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:04:06.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:04:06.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:04:06.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: starting mon.smithi119 rank 0 at public addrs [v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0] at bind addrs [v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi119 fsid 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:06.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: mon.smithi119@-1(???) e1 preinit fsid 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:06.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: mon.smithi119@-1(???).mds e1 new map 2024-04-22T15:04:06.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: mon.smithi119@-1(???).mds e1 print_map 2024-04-22T15:04:06.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: e1 2024-04-22T15:04:06.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T15:04:06.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T15:04:06.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: legacy client fscid: -1 2024-04-22T15:04:06.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:04:06.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: No filesystems configured 2024-04-22T15:04:06.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: mon.smithi119@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-22T15:04:06.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: mon.smithi119@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:04:06.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: mon.smithi119@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:04:06.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: mon.smithi119@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:04:06.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: mon.smithi119@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-22T15:04:06.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: mon.smithi119 is new leader, mons smithi119 in quorum (ranks 0) 2024-04-22T15:04:06.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: monmap e1: 1 mons at {smithi119=[v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0]} 2024-04-22T15:04:06.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: fsmap 2024-04-22T15:04:06.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T15:04:06.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 ceph-mon[21835]: mgrmap e1: no daemons active 2024-04-22T15:04:06.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 bash[21816]: e31b6591d879b56cac373b321e58fad7bc689660f99a38872f480090f3e06685 2024-04-22T15:04:06.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:06 smithi119 systemd[1]: Started Ceph mon.smithi119 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:04:07.672 INFO:teuthology.orchestra.run.smithi119.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-22T15:04:07.673 INFO:teuthology.orchestra.run.smithi119.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:04:07.673 INFO:teuthology.orchestra.run.smithi119.stderr:Creating mgr... 2024-04-22T15:04:07.673 INFO:teuthology.orchestra.run.smithi119.stderr:Verifying port 9283 ... 2024-04-22T15:04:07.891 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: Failed to reset failed state of unit ceph-74739db6-00b9-11ef-bc93-c7b262605968@mgr.smithi119.jjbxxd.service: Unit ceph-74739db6-00b9-11ef-bc93-c7b262605968@mgr.smithi119.jjbxxd.service not loaded. 2024-04-22T15:04:07.898 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: Created symlink /etc/systemd/system/ceph-74739db6-00b9-11ef-bc93-c7b262605968.target.wants/ceph-74739db6-00b9-11ef-bc93-c7b262605968@mgr.smithi119.jjbxxd.service → /etc/systemd/system/ceph-74739db6-00b9-11ef-bc93-c7b262605968@.service. 2024-04-22T15:04:08.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:08 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/142372834' entity='client.admin' 2024-04-22T15:04:09.032 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: disabled 2024-04-22T15:04:09.040 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: inactive 2024-04-22T15:04:09.041 INFO:teuthology.orchestra.run.smithi119.stderr:firewalld.service is not enabled 2024-04-22T15:04:09.041 INFO:teuthology.orchestra.run.smithi119.stderr:Not possible to enable service . firewalld.service is not available 2024-04-22T15:04:09.051 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: disabled 2024-04-22T15:04:09.059 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: inactive 2024-04-22T15:04:09.060 INFO:teuthology.orchestra.run.smithi119.stderr:firewalld.service is not enabled 2024-04-22T15:04:09.060 INFO:teuthology.orchestra.run.smithi119.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-22T15:04:09.060 INFO:teuthology.orchestra.run.smithi119.stderr:Waiting for mgr to start... 2024-04-22T15:04:09.060 INFO:teuthology.orchestra.run.smithi119.stderr:Waiting for mgr... 2024-04-22T15:04:09.616 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 2024-04-22T15:04:09.616 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: { 2024-04-22T15:04:09.616 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "fsid": "74739db6-00b9-11ef-bc93-c7b262605968", 2024-04-22T15:04:09.616 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "health": { 2024-04-22T15:04:09.616 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T15:04:09.616 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T15:04:09.616 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T15:04:09.616 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 0 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "smithi119" 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "quorum_age": 3, 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:09.617 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T15:04:09.618 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:09.618 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T15:04:09.618 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T15:04:09.618 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T15:04:09.618 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T15:04:09.622 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T15:04:09.622 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T15:04:09.622 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:09.622 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T15:04:09.622 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T15:04:09.622 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T15:04:09.622 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T15:04:09.623 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T15:04:09.623 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T15:04:09.623 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T15:04:09.623 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T15:04:09.623 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T15:04:09.623 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:09.623 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T15:04:09.623 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:09.623 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T15:04:09.623 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T15:04:09.623 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:09.623 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "available": false, 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "iostat", 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "nfs", 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "restful" 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "services": {} 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "modified": "2024-04-22T15:03:56.134395+0000", 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "services": {} 2024-04-22T15:04:09.624 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:09.625 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T15:04:09.625 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: } 2024-04-22T15:04:09.960 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:09 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/540137807' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T15:04:10.246 INFO:teuthology.orchestra.run.smithi119.stderr:mgr not available, waiting (1/15)... 2024-04-22T15:04:12.756 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 2024-04-22T15:04:12.756 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: { 2024-04-22T15:04:12.756 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "fsid": "74739db6-00b9-11ef-bc93-c7b262605968", 2024-04-22T15:04:12.756 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "health": { 2024-04-22T15:04:12.756 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T15:04:12.756 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T15:04:12.756 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 0 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "smithi119" 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T15:04:12.757 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:12.758 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T15:04:12.758 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T15:04:12.758 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T15:04:12.758 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T15:04:12.758 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T15:04:12.758 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T15:04:12.758 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:12.758 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T15:04:12.758 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T15:04:12.759 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T15:04:12.759 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T15:04:12.759 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T15:04:12.759 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T15:04:12.759 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T15:04:12.759 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T15:04:12.759 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T15:04:12.759 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:12.759 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T15:04:12.759 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:12.759 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "available": false, 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "iostat", 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "nfs", 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "restful" 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "services": {} 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:12.760 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T15:04:12.761 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:12.761 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "modified": "2024-04-22T15:03:56.134395+0000", 2024-04-22T15:04:12.761 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "services": {} 2024-04-22T15:04:12.761 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:12.761 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T15:04:12.761 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: } 2024-04-22T15:04:12.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:12 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/595867347' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T15:04:13.567 INFO:teuthology.orchestra.run.smithi119.stderr:mgr not available, waiting (2/15)... 2024-04-22T15:04:15.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: Activating manager daemon smithi119.jjbxxd 2024-04-22T15:04:15.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: mgrmap e2: smithi119.jjbxxd(active, starting, since 0.00292705s) 2024-04-22T15:04:15.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: from='mgr.14100 172.21.15.119:0/1162689037' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:04:15.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: from='mgr.14100 172.21.15.119:0/1162689037' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:04:15.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: from='mgr.14100 172.21.15.119:0/1162689037' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:04:15.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: from='mgr.14100 172.21.15.119:0/1162689037' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:04:15.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: from='mgr.14100 172.21.15.119:0/1162689037' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi119.jjbxxd", "id": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:04:15.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: Manager daemon smithi119.jjbxxd is now available 2024-04-22T15:04:15.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: from='mgr.14100 172.21.15.119:0/1162689037' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:04:15.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: from='mgr.14100 172.21.15.119:0/1162689037' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:04:15.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: from='mgr.14100 172.21.15.119:0/1162689037' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:15.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: from='mgr.14100 172.21.15.119:0/1162689037' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:15.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:14 smithi119 ceph-mon[21835]: from='mgr.14100 172.21.15.119:0/1162689037' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:16.225 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 2024-04-22T15:04:16.225 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: { 2024-04-22T15:04:16.225 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "fsid": "74739db6-00b9-11ef-bc93-c7b262605968", 2024-04-22T15:04:16.225 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "health": { 2024-04-22T15:04:16.225 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 0 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "smithi119" 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:16.226 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T15:04:16.227 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T15:04:16.227 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:16.227 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T15:04:16.227 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:16.227 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T15:04:16.227 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T15:04:16.227 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T15:04:16.227 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T15:04:16.230 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T15:04:16.230 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T15:04:16.230 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T15:04:16.231 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "available": true, 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "iostat", 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "nfs", 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "restful" 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "services": {} 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "modified": "2024-04-22T15:03:56.134395+0000", 2024-04-22T15:04:16.232 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "services": {} 2024-04-22T15:04:16.233 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:16.233 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T15:04:16.237 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: } 2024-04-22T15:04:16.807 INFO:teuthology.orchestra.run.smithi119.stderr:mgr is available 2024-04-22T15:04:16.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:16 smithi119 ceph-mon[21835]: mgrmap e3: smithi119.jjbxxd(active, since 1.00575s) 2024-04-22T15:04:16.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:16 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/379420538' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T15:04:17.389 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 2024-04-22T15:04:17.389 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: [global] 2024-04-22T15:04:17.389 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: fsid = 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:17.389 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-22T15:04:17.389 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-22T15:04:17.390 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-22T15:04:17.390 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-22T15:04:17.390 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 2024-04-22T15:04:17.390 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: [mgr] 2024-04-22T15:04:17.390 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-22T15:04:17.390 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 2024-04-22T15:04:17.390 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: [osd] 2024-04-22T15:04:17.390 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-22T15:04:17.390 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-22T15:04:18.035 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:17 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/1360328694' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T15:04:18.035 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:17 smithi119 ceph-mon[21835]: mgrmap e4: smithi119.jjbxxd(active, since 2s) 2024-04-22T15:04:18.038 INFO:teuthology.orchestra.run.smithi119.stderr:Enabling cephadm module... 2024-04-22T15:04:19.211 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:18 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3831558600' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-22T15:04:20.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:19 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3831558600' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-22T15:04:20.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:19 smithi119 ceph-mon[21835]: mgrmap e5: smithi119.jjbxxd(active, since 4s) 2024-04-22T15:04:20.284 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: { 2024-04-22T15:04:20.284 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-22T15:04:20.284 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "available": true, 2024-04-22T15:04:20.284 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "active_name": "smithi119.jjbxxd", 2024-04-22T15:04:20.285 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-22T15:04:20.285 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: } 2024-04-22T15:04:20.885 INFO:teuthology.orchestra.run.smithi119.stderr:Waiting for the mgr to restart... 2024-04-22T15:04:20.885 INFO:teuthology.orchestra.run.smithi119.stderr:Waiting for mgr epoch 5... 2024-04-22T15:04:21.013 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:20 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/4045980167' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T15:04:26.036 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: { 2024-04-22T15:04:26.036 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-22T15:04:26.036 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "initialized": true 2024-04-22T15:04:26.036 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: } 2024-04-22T15:04:26.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: Active manager daemon smithi119.jjbxxd restarted 2024-04-22T15:04:26.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: Activating manager daemon smithi119.jjbxxd 2024-04-22T15:04:26.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: osdmap e2: 0 total, 0 up, 0 in 2024-04-22T15:04:26.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: mgrmap e6: smithi119.jjbxxd(active, starting, since 0.0525162s) 2024-04-22T15:04:26.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi119.jjbxxd", "id": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: Manager daemon smithi119.jjbxxd is now available 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: [22/Apr/2024:15:04:25] ENGINE Bus STARTING 2024-04-22T15:04:26.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: [22/Apr/2024:15:04:25] ENGINE Serving on https://172.21.15.119:7150 2024-04-22T15:04:26.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: [22/Apr/2024:15:04:25] ENGINE Bus STARTED 2024-04-22T15:04:26.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:26.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:25 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:04:26.540 INFO:teuthology.orchestra.run.smithi119.stderr:mgr epoch 5 is available 2024-04-22T15:04:26.540 INFO:teuthology.orchestra.run.smithi119.stderr:Setting orchestrator backend to cephadm... 2024-04-22T15:04:27.249 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:27 smithi119 ceph-mon[21835]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-22T15:04:27.249 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:27 smithi119 ceph-mon[21835]: mgrmap e7: smithi119.jjbxxd(active, since 1.05477s) 2024-04-22T15:04:27.249 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:27 smithi119 ceph-mon[21835]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-22T15:04:28.361 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:28 smithi119 ceph-mon[21835]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:28.361 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:28 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:28.361 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:28 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:04:28.361 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:28 smithi119 ceph-mon[21835]: mgrmap e8: smithi119.jjbxxd(active, since 2s) 2024-04-22T15:04:28.362 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: value unchanged 2024-04-22T15:04:28.961 INFO:teuthology.orchestra.run.smithi119.stderr:Generating ssh key... 2024-04-22T15:04:29.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:29 smithi119 ceph-mon[21835]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:31.055 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:30 smithi119 ceph-mon[21835]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:31.056 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:30 smithi119 ceph-mon[21835]: Generating ssh key... 2024-04-22T15:04:31.056 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:30 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:31.056 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:30 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:31.056 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDeToV5EVxI7+V8iPQTi8guXBrPl5NIPzANyTlWp6/MVqZpU5GlNlhub2TS6YE3EwpxHxT6qc/1Zu8uhk+7w/FaIZfJsKIxBbWGcQ1TI3OOkLqVaImSEdS/ZoHpNLmPfByykD9hlaF+8+briUbPLRYOhSqwjV6a4hETTvNizKbrY9m5dD5L88rA7K9Nd1COHiYxBaSgHoe2W+x5BzFsKnWHBZHKYDm73vsrgonct/Le1LhbEVmG2ER5n4HChqaCTxJ4Gq5pbFVXJ/Pw5H3OJ/yvIxYi4jIGM0bTR2/HE6ggLuT+mrzuKRdwf2UPVAyPsfb+ILOF5t3XedVkPC5DKHtFhUfJEzzH17LFSycNOVsoin4Bpc0b9l4mB0DEqYJG1lcPPsbziZq4+EPHqM4R2KJ1AZTlzNbQKCpgu9IW+lE3+7yp/gE2LMeqC3wN3HXlwnmwutFCg6rpJGPUD9wkG0EyaYbnLzEYGehV8U9UuAHRN5a17QGbejd/vGStekM5Avk= ceph-74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:31.650 INFO:teuthology.orchestra.run.smithi119.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-22T15:04:31.650 INFO:teuthology.orchestra.run.smithi119.stderr:Adding key to root@localhost authorized_keys... 2024-04-22T15:04:31.650 INFO:teuthology.orchestra.run.smithi119.stderr:Adding host smithi119... 2024-04-22T15:04:32.567 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:32 smithi119 ceph-mon[21835]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:33.115 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: Added host 'smithi119' with addr '172.21.15.119' 2024-04-22T15:04:33.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:33 smithi119 ceph-mon[21835]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi119", "addr": "172.21.15.119", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:33.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:33 smithi119 ceph-mon[21835]: Deploying cephadm binary to smithi119 2024-04-22T15:04:33.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:33 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:33.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:33 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:04:33.689 INFO:teuthology.orchestra.run.smithi119.stderr:Deploying mon service with default placement... 2024-04-22T15:04:34.387 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-22T15:04:34.706 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:34 smithi119 ceph-mon[21835]: Added host smithi119 2024-04-22T15:04:34.706 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:34 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:35.093 INFO:teuthology.orchestra.run.smithi119.stderr:Deploying mgr service with default placement... 2024-04-22T15:04:35.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:35 smithi119 ceph-mon[21835]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:35.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:35 smithi119 ceph-mon[21835]: Saving service mon spec with placement count:5 2024-04-22T15:04:35.854 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-22T15:04:37.176 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:36 smithi119 ceph-mon[21835]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:37.177 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:36 smithi119 ceph-mon[21835]: Saving service mgr spec with placement count:2 2024-04-22T15:04:37.177 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:36 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:37.177 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:36 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:37.178 INFO:teuthology.orchestra.run.smithi119.stderr:Deploying crash service with default placement... 2024-04-22T15:04:37.832 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: Scheduled crash update... 2024-04-22T15:04:38.457 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:38 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:38.481 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:38 smithi119 ceph-mon[21835]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:38.481 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:38 smithi119 ceph-mon[21835]: Saving service crash spec with placement * 2024-04-22T15:04:38.481 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:38 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:39.058 INFO:teuthology.orchestra.run.smithi119.stderr:Deploying prometheus service with default placement... 2024-04-22T15:04:39.668 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:39 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:39.668 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:39 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:39.731 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-04-22T15:04:40.316 INFO:teuthology.orchestra.run.smithi119.stderr:Deploying grafana service with default placement... 2024-04-22T15:04:40.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:40 smithi119 ceph-mon[21835]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:40.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:40 smithi119 ceph-mon[21835]: Saving service prometheus spec with placement count:1 2024-04-22T15:04:40.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:40 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:41.847 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-04-22T15:04:42.451 INFO:teuthology.orchestra.run.smithi119.stderr:Deploying node-exporter service with default placement... 2024-04-22T15:04:42.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:42 smithi119 ceph-mon[21835]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:42.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:42 smithi119 ceph-mon[21835]: Saving service grafana spec with placement count:1 2024-04-22T15:04:42.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:42 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:43.138 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-04-22T15:04:43.707 INFO:teuthology.orchestra.run.smithi119.stderr:Deploying alertmanager service with default placement... 2024-04-22T15:04:44.304 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:44 smithi119 ceph-mon[21835]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:44.304 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:44 smithi119 ceph-mon[21835]: Saving service node-exporter spec with placement * 2024-04-22T15:04:44.304 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:44 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:44.304 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-04-22T15:04:45.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:45 smithi119 ceph-mon[21835]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:45.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:45 smithi119 ceph-mon[21835]: Saving service alertmanager spec with placement count:1 2024-04-22T15:04:45.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:45 smithi119 ceph-mon[21835]: from='mgr.14118 172.21.15.119:0/1263253385' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:46.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:46 smithi119 ceph-mon[21835]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:04:46.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:46 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/1688701346' entity='client.admin' 2024-04-22T15:04:47.427 INFO:teuthology.orchestra.run.smithi119.stderr:Enabling the dashboard module... 2024-04-22T15:04:47.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:47 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2067341195' entity='client.admin' 2024-04-22T15:04:49.044 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:48 smithi119 ceph-mon[21835]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:04:49.044 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:48 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3403290125' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-22T15:04:50.105 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:49 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3403290125' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-22T15:04:50.106 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:49 smithi119 ceph-mon[21835]: mgrmap e9: smithi119.jjbxxd(active, since 23s) 2024-04-22T15:04:50.106 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: { 2024-04-22T15:04:50.106 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-22T15:04:50.107 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "available": true, 2024-04-22T15:04:50.107 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "active_name": "smithi119.jjbxxd", 2024-04-22T15:04:50.107 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-22T15:04:50.107 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: } 2024-04-22T15:04:50.708 INFO:teuthology.orchestra.run.smithi119.stderr:Waiting for the mgr to restart... 2024-04-22T15:04:50.708 INFO:teuthology.orchestra.run.smithi119.stderr:Waiting for mgr epoch 9... 2024-04-22T15:04:50.964 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:50 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3129948217' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T15:04:55.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:54 smithi119 ceph-mon[21835]: Active manager daemon smithi119.jjbxxd restarted 2024-04-22T15:04:55.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:54 smithi119 ceph-mon[21835]: Activating manager daemon smithi119.jjbxxd 2024-04-22T15:04:55.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:54 smithi119 ceph-mon[21835]: osdmap e3: 0 total, 0 up, 0 in 2024-04-22T15:04:55.830 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: { 2024-04-22T15:04:55.831 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-22T15:04:55.831 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: "initialized": true 2024-04-22T15:04:55.831 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: } 2024-04-22T15:04:55.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: mgrmap e10: smithi119.jjbxxd(active, starting, since 0.0557377s) 2024-04-22T15:04:55.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:04:55.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi119.jjbxxd", "id": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:04:55.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:04:55.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:04:55.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:04:55.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: Manager daemon smithi119.jjbxxd is now available 2024-04-22T15:04:55.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:04:55.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:04:55.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:04:55.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:04:55.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: [22/Apr/2024:15:04:55] ENGINE Bus STARTING 2024-04-22T15:04:55.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: [22/Apr/2024:15:04:55] ENGINE Serving on https://172.21.15.119:7150 2024-04-22T15:04:55.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: [22/Apr/2024:15:04:55] ENGINE Bus STARTED 2024-04-22T15:04:55.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:55 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:56.543 INFO:teuthology.orchestra.run.smithi119.stderr:mgr epoch 9 is available 2024-04-22T15:04:56.543 INFO:teuthology.orchestra.run.smithi119.stderr:Generating a dashboard self-signed certificate... 2024-04-22T15:04:57.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:56 smithi119 ceph-mon[21835]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-22T15:04:57.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:56 smithi119 ceph-mon[21835]: mgrmap e11: smithi119.jjbxxd(active, since 1.0659s) 2024-04-22T15:04:57.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:56 smithi119 ceph-mon[21835]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-22T15:04:57.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:56 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:57.513 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-22T15:04:58.149 INFO:teuthology.orchestra.run.smithi119.stderr:Creating initial admin user... 2024-04-22T15:04:58.736 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:58 smithi119 ceph-mon[21835]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:58.736 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:58 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:58.736 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:58 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:58.736 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:58 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:04:58.737 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:04:58 smithi119 ceph-mon[21835]: mgrmap e12: smithi119.jjbxxd(active, since 2s) 2024-04-22T15:04:59.372 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$xeqws2ro...PTMrJHK3u3OMiB4419xCZW9XT45uLDcB2HSnThxxM6", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1713798299, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-22T15:05:00.432 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:00 smithi119 ceph-mon[21835]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:00.432 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:00 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:00.435 INFO:teuthology.orchestra.run.smithi119.stderr:Fetching dashboard port number... 2024-04-22T15:05:01.033 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: 8443 2024-04-22T15:05:01.648 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:01 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2227796277' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-22T15:05:01.660 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: disabled 2024-04-22T15:05:01.666 INFO:teuthology.orchestra.run.smithi119.stderr:systemctl: inactive 2024-04-22T15:05:01.667 INFO:teuthology.orchestra.run.smithi119.stderr:firewalld.service is not enabled 2024-04-22T15:05:01.667 INFO:teuthology.orchestra.run.smithi119.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-22T15:05:01.668 INFO:teuthology.orchestra.run.smithi119.stderr:Ceph Dashboard is now available at: 2024-04-22T15:05:01.669 INFO:teuthology.orchestra.run.smithi119.stderr: 2024-04-22T15:05:01.669 INFO:teuthology.orchestra.run.smithi119.stderr: URL: https://smithi119:8443/ 2024-04-22T15:05:01.669 INFO:teuthology.orchestra.run.smithi119.stderr: User: admin 2024-04-22T15:05:01.669 INFO:teuthology.orchestra.run.smithi119.stderr: Password: dfw3xft8xh 2024-04-22T15:05:01.669 INFO:teuthology.orchestra.run.smithi119.stderr: 2024-04-22T15:05:01.669 INFO:teuthology.orchestra.run.smithi119.stderr:Enabling autotune for osd_memory_target 2024-04-22T15:05:02.717 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:02 smithi119 ceph-mon[21835]: mgrmap e13: smithi119.jjbxxd(active, since 6s) 2024-04-22T15:05:03.711 INFO:teuthology.orchestra.run.smithi119.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-22T15:05:04.330 INFO:teuthology.orchestra.run.smithi119.stderr:You can access the Ceph CLI with: 2024-04-22T15:05:04.330 INFO:teuthology.orchestra.run.smithi119.stderr: 2024-04-22T15:05:04.330 INFO:teuthology.orchestra.run.smithi119.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:05:04.330 INFO:teuthology.orchestra.run.smithi119.stderr: 2024-04-22T15:05:04.330 INFO:teuthology.orchestra.run.smithi119.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-22T15:05:04.331 INFO:teuthology.orchestra.run.smithi119.stderr: 2024-04-22T15:05:04.331 INFO:teuthology.orchestra.run.smithi119.stderr: ceph telemetry on 2024-04-22T15:05:04.331 INFO:teuthology.orchestra.run.smithi119.stderr: 2024-04-22T15:05:04.331 INFO:teuthology.orchestra.run.smithi119.stderr:For more information see: 2024-04-22T15:05:04.331 INFO:teuthology.orchestra.run.smithi119.stderr: 2024-04-22T15:05:04.331 INFO:teuthology.orchestra.run.smithi119.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-22T15:05:04.331 INFO:teuthology.orchestra.run.smithi119.stderr: 2024-04-22T15:05:04.331 INFO:teuthology.orchestra.run.smithi119.stderr:Bootstrap complete. 2024-04-22T15:05:04.366 INFO:tasks.cephadm:Fetching config... 2024-04-22T15:05:04.367 DEBUG:teuthology.orchestra.run.smithi119:> set -ex 2024-04-22T15:05:04.367 DEBUG:teuthology.orchestra.run.smithi119:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-22T15:05:04.382 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-22T15:05:04.382 DEBUG:teuthology.orchestra.run.smithi119:> set -ex 2024-04-22T15:05:04.382 DEBUG:teuthology.orchestra.run.smithi119:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-22T15:05:04.437 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-22T15:05:04.437 DEBUG:teuthology.orchestra.run.smithi119:> set -ex 2024-04-22T15:05:04.437 DEBUG:teuthology.orchestra.run.smithi119:> sudo dd if=/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/keyring of=/dev/stdout 2024-04-22T15:05:04.500 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-22T15:05:04.500 DEBUG:teuthology.orchestra.run.smithi119:> set -ex 2024-04-22T15:05:04.500 DEBUG:teuthology.orchestra.run.smithi119:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-22T15:05:04.555 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-22T15:05:04.556 DEBUG:teuthology.orchestra.run.smithi119:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDeToV5EVxI7+V8iPQTi8guXBrPl5NIPzANyTlWp6/MVqZpU5GlNlhub2TS6YE3EwpxHxT6qc/1Zu8uhk+7w/FaIZfJsKIxBbWGcQ1TI3OOkLqVaImSEdS/ZoHpNLmPfByykD9hlaF+8+briUbPLRYOhSqwjV6a4hETTvNizKbrY9m5dD5L88rA7K9Nd1COHiYxBaSgHoe2W+x5BzFsKnWHBZHKYDm73vsrgonct/Le1LhbEVmG2ER5n4HChqaCTxJ4Gq5pbFVXJ/Pw5H3OJ/yvIxYi4jIGM0bTR2/HE6ggLuT+mrzuKRdwf2UPVAyPsfb+ILOF5t3XedVkPC5DKHtFhUfJEzzH17LFSycNOVsoin4Bpc0b9l4mB0DEqYJG1lcPPsbziZq4+EPHqM4R2KJ1AZTlzNbQKCpgu9IW+lE3+7yp/gE2LMeqC3wN3HXlwnmwutFCg6rpJGPUD9wkG0EyaYbnLzEYGehV8U9UuAHRN5a17QGbejd/vGStekM5Avk= ceph-74739db6-00b9-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-22T15:05:04.648 INFO:teuthology.orchestra.run.smithi119.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDeToV5EVxI7+V8iPQTi8guXBrPl5NIPzANyTlWp6/MVqZpU5GlNlhub2TS6YE3EwpxHxT6qc/1Zu8uhk+7w/FaIZfJsKIxBbWGcQ1TI3OOkLqVaImSEdS/ZoHpNLmPfByykD9hlaF+8+briUbPLRYOhSqwjV6a4hETTvNizKbrY9m5dD5L88rA7K9Nd1COHiYxBaSgHoe2W+x5BzFsKnWHBZHKYDm73vsrgonct/Le1LhbEVmG2ER5n4HChqaCTxJ4Gq5pbFVXJ/Pw5H3OJ/yvIxYi4jIGM0bTR2/HE6ggLuT+mrzuKRdwf2UPVAyPsfb+ILOF5t3XedVkPC5DKHtFhUfJEzzH17LFSycNOVsoin4Bpc0b9l4mB0DEqYJG1lcPPsbziZq4+EPHqM4R2KJ1AZTlzNbQKCpgu9IW+lE3+7yp/gE2LMeqC3wN3HXlwnmwutFCg6rpJGPUD9wkG0EyaYbnLzEYGehV8U9UuAHRN5a17QGbejd/vGStekM5Avk= ceph-74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:05:04.659 DEBUG:teuthology.orchestra.run.smithi186:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDeToV5EVxI7+V8iPQTi8guXBrPl5NIPzANyTlWp6/MVqZpU5GlNlhub2TS6YE3EwpxHxT6qc/1Zu8uhk+7w/FaIZfJsKIxBbWGcQ1TI3OOkLqVaImSEdS/ZoHpNLmPfByykD9hlaF+8+briUbPLRYOhSqwjV6a4hETTvNizKbrY9m5dD5L88rA7K9Nd1COHiYxBaSgHoe2W+x5BzFsKnWHBZHKYDm73vsrgonct/Le1LhbEVmG2ER5n4HChqaCTxJ4Gq5pbFVXJ/Pw5H3OJ/yvIxYi4jIGM0bTR2/HE6ggLuT+mrzuKRdwf2UPVAyPsfb+ILOF5t3XedVkPC5DKHtFhUfJEzzH17LFSycNOVsoin4Bpc0b9l4mB0DEqYJG1lcPPsbziZq4+EPHqM4R2KJ1AZTlzNbQKCpgu9IW+lE3+7yp/gE2LMeqC3wN3HXlwnmwutFCg6rpJGPUD9wkG0EyaYbnLzEYGehV8U9UuAHRN5a17QGbejd/vGStekM5Avk= ceph-74739db6-00b9-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-22T15:05:04.702 INFO:teuthology.orchestra.run.smithi186.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDeToV5EVxI7+V8iPQTi8guXBrPl5NIPzANyTlWp6/MVqZpU5GlNlhub2TS6YE3EwpxHxT6qc/1Zu8uhk+7w/FaIZfJsKIxBbWGcQ1TI3OOkLqVaImSEdS/ZoHpNLmPfByykD9hlaF+8+briUbPLRYOhSqwjV6a4hETTvNizKbrY9m5dD5L88rA7K9Nd1COHiYxBaSgHoe2W+x5BzFsKnWHBZHKYDm73vsrgonct/Le1LhbEVmG2ER5n4HChqaCTxJ4Gq5pbFVXJ/Pw5H3OJ/yvIxYi4jIGM0bTR2/HE6ggLuT+mrzuKRdwf2UPVAyPsfb+ILOF5t3XedVkPC5DKHtFhUfJEzzH17LFSycNOVsoin4Bpc0b9l4mB0DEqYJG1lcPPsbziZq4+EPHqM4R2KJ1AZTlzNbQKCpgu9IW+lE3+7yp/gE2LMeqC3wN3HXlwnmwutFCg6rpJGPUD9wkG0EyaYbnLzEYGehV8U9UuAHRN5a17QGbejd/vGStekM5Avk= ceph-74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:05:04.714 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-22T15:05:04.846 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:04 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/464364620' entity='client.admin' 2024-04-22T15:05:08.440 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-22T15:05:08.440 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-22T15:05:09.094 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:08 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2774555261' entity='client.admin' 2024-04-22T15:05:11.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:11 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:11.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:11 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi119", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:05:11.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:11 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:11.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:11 smithi119 ceph-mon[21835]: Deploying daemon alertmanager.smithi119 on smithi119 2024-04-22T15:05:11.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:11 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:11.530 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi186 2024-04-22T15:05:11.530 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-04-22T15:05:11.531 DEBUG:teuthology.orchestra.run.smithi186:> dd of=/etc/ceph/ceph.conf 2024-04-22T15:05:11.549 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-04-22T15:05:11.549 DEBUG:teuthology.orchestra.run.smithi186:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:05:11.607 INFO:tasks.cephadm:Adding host smithi186 to orchestrator... 2024-04-22T15:05:11.607 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch host add smithi186 2024-04-22T15:05:12.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:12 smithi119 ceph-mon[21835]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:16.048 INFO:teuthology.orchestra.run.smithi119.stdout:Added host 'smithi186' with addr '172.21.15.186' 2024-04-22T15:05:16.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:15 smithi119 ceph-mon[21835]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:16.245 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:15 smithi119 ceph-mon[21835]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi186", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:16.245 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:15 smithi119 ceph-mon[21835]: Deploying cephadm binary to smithi186 2024-04-22T15:05:16.964 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-22T15:05:17.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:17 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:17.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:17 smithi119 ceph-mon[21835]: Added host smithi186 2024-04-22T15:05:18.565 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:18 smithi119 ceph-mon[21835]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:20.453 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:05:20.453 INFO:teuthology.orchestra.run.smithi119.stdout:[{"addr": "172.21.15.119", "hostname": "smithi119", "labels": [], "status": ""}, {"addr": "172.21.15.186", "hostname": "smithi186", "labels": [], "status": ""}] 2024-04-22T15:05:20.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:20 smithi119 ceph-mon[21835]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:21.460 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-22T15:05:21.461 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-22T15:05:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:21 smithi119 ceph-mon[21835]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:05:22.962 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:22 smithi119 ceph-mon[21835]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:22.962 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:22 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:22.962 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:22 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi119", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:05:22.962 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:22 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi119", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-22T15:05:22.962 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:22 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:22.963 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:22 smithi119 ceph-mon[21835]: Deploying daemon crash.smithi119 on smithi119 2024-04-22T15:05:23.618 INFO:teuthology.orchestra.run.smithi119.stderr:adjusted tunables profile to default 2024-04-22T15:05:23.958 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:23 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2556227339' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-22T15:05:24.546 INFO:tasks.cephadm:Adding mon.smithi119 on smithi119 2024-04-22T15:05:24.546 INFO:tasks.cephadm:Adding mon.smithi186 on smithi186 2024-04-22T15:05:24.546 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch apply mon '2;smithi119:172.21.15.119=smithi119;smithi186:172.21.15.186=smithi186' 2024-04-22T15:05:24.953 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:24 smithi119 ceph-mon[21835]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:24.953 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:24 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2556227339' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-22T15:05:24.953 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:24 smithi119 ceph-mon[21835]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T15:05:26.038 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:25 smithi119 ceph-mon[21835]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:26.038 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:25 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:26.298 INFO:teuthology.orchestra.run.smithi186.stdout:Scheduled mon update... 2024-04-22T15:05:27.024 DEBUG:teuthology.orchestra.run.smithi186:mon.smithi186> sudo journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi186.service 2024-04-22T15:05:27.026 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T15:05:27.026 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:05:27.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:26 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:27.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:26 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:27.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:26 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:27.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:26 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:05:27.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:26 smithi119 ceph-mon[21835]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:05:27.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:26 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:27.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:26 smithi119 ceph-mon[21835]: Deploying daemon grafana.smithi119 on smithi119 2024-04-22T15:05:27.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:26 smithi119 ceph-mon[21835]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi119:172.21.15.119=smithi119;smithi186:172.21.15.186=smithi186", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:27.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:26 smithi119 ceph-mon[21835]: Saving service mon spec with placement smithi119:172.21.15.119=smithi119;smithi186:172.21.15.186=smithi186;count:2 2024-04-22T15:05:27.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:26 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:28.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:28 smithi119 ceph-mon[21835]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:29.067 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T15:05:29.067 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":1,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:03:50.504762Z","created":"2024-04-22T15:03:50.504762Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi119","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:3300","nonce":0},{"type":"v1","addr":"172.21.15.119:6789","nonce":0}]},"addr":"172.21.15.119:6789/0","public_addr":"172.21.15.119:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T15:05:29.069 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 1 2024-04-22T15:05:29.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:29 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/316883395' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:30.599 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T15:05:30.600 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:05:30.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:30 smithi119 ceph-mon[21835]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:30.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:30 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:32.366 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T15:05:32.366 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":1,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:03:50.504762Z","created":"2024-04-22T15:03:50.504762Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi119","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:3300","nonce":0},{"type":"v1","addr":"172.21.15.119:6789","nonce":0}]},"addr":"172.21.15.119:6789/0","public_addr":"172.21.15.119:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T15:05:32.368 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 1 2024-04-22T15:05:32.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:32 smithi119 ceph-mon[21835]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:32.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:32 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/3769942731' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:33.961 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T15:05:33.962 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:05:34.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:34 smithi119 ceph-mon[21835]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:35.640 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T15:05:35.640 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":1,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:03:50.504762Z","created":"2024-04-22T15:03:50.504762Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi119","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:3300","nonce":0},{"type":"v1","addr":"172.21.15.119:6789","nonce":0}]},"addr":"172.21.15.119:6789/0","public_addr":"172.21.15.119:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T15:05:35.642 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 1 2024-04-22T15:05:36.708 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:36 smithi119 ceph-mon[21835]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:36.708 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:36 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/821602722' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:37.115 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T15:05:37.116 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:05:38.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:38 smithi119 ceph-mon[21835]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:38.970 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T15:05:38.970 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":1,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:03:50.504762Z","created":"2024-04-22T15:03:50.504762Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi119","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:3300","nonce":0},{"type":"v1","addr":"172.21.15.119:6789","nonce":0}]},"addr":"172.21.15.119:6789/0","public_addr":"172.21.15.119:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T15:05:38.972 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 1 2024-04-22T15:05:39.654 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:39 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/1640141278' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:39.654 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:39 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:40.550 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T15:05:40.551 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:05:40.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:40 smithi119 ceph-mon[21835]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:40.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:40 smithi119 ceph-mon[21835]: Deploying daemon node-exporter.smithi119 on smithi119 2024-04-22T15:05:40.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:40 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:42.420 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T15:05:42.421 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":1,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:03:50.504762Z","created":"2024-04-22T15:03:50.504762Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi119","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:3300","nonce":0},{"type":"v1","addr":"172.21.15.119:6789","nonce":0}]},"addr":"172.21.15.119:6789/0","public_addr":"172.21.15.119:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T15:05:42.422 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 1 2024-04-22T15:05:42.592 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:42 smithi119 ceph-mon[21835]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:42.592 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:42 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/1163516188' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:43.959 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:43 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:43.959 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:43 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-22T15:05:44.019 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T15:05:44.019 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:05:44.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:44 smithi119 ceph-mon[21835]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-22T15:05:44.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:44 smithi119 ceph-mon[21835]: mgrmap e14: smithi119.jjbxxd(active, since 48s) 2024-04-22T15:05:45.699 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T15:05:45.700 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":1,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:03:50.504762Z","created":"2024-04-22T15:03:50.504762Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi119","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:3300","nonce":0},{"type":"v1","addr":"172.21.15.119:6789","nonce":0}]},"addr":"172.21.15.119:6789/0","public_addr":"172.21.15.119:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T15:05:45.702 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 1 2024-04-22T15:05:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:45 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/4155918606' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:47.257 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T15:05:47.257 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:05:49.011 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T15:05:49.011 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":1,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:03:50.504762Z","created":"2024-04-22T15:03:50.504762Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi119","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:3300","nonce":0},{"type":"v1","addr":"172.21.15.119:6789","nonce":0}]},"addr":"172.21.15.119:6789/0","public_addr":"172.21.15.119:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T15:05:49.013 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 1 2024-04-22T15:05:49.321 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:49 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/4149242112' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:50.618 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T15:05:50.618 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:05:50.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: Active manager daemon smithi119.jjbxxd restarted 2024-04-22T15:05:50.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: Activating manager daemon smithi119.jjbxxd 2024-04-22T15:05:50.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: osdmap e5: 0 total, 0 up, 0 in 2024-04-22T15:05:50.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: mgrmap e15: smithi119.jjbxxd(active, starting, since 0.387435s) 2024-04-22T15:05:50.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi119.jjbxxd", "id": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: Manager daemon smithi119.jjbxxd is now available 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:05:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:51.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:51 smithi119 ceph-mon[21835]: [22/Apr/2024:15:05:50] ENGINE Bus STARTING 2024-04-22T15:05:51.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:51 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:51.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:51 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:51.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:51 smithi119 ceph-mon[21835]: mgrmap e16: smithi119.jjbxxd(active, since 1.39028s) 2024-04-22T15:05:52.502 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T15:05:52.502 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":1,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:03:50.504762Z","created":"2024-04-22T15:03:50.504762Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi119","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:3300","nonce":0},{"type":"v1","addr":"172.21.15.119:6789","nonce":0}]},"addr":"172.21.15.119:6789/0","public_addr":"172.21.15.119:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T15:05:52.504 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 1 2024-04-22T15:05:52.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:52 smithi119 ceph-mon[21835]: [22/Apr/2024:15:05:50] ENGINE Serving on https://172.21.15.119:7150 2024-04-22T15:05:52.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:52 smithi119 ceph-mon[21835]: [22/Apr/2024:15:05:50] ENGINE Bus STARTED 2024-04-22T15:05:52.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:52 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:52.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:52 smithi119 ceph-mon[21835]: mgrmap e17: smithi119.jjbxxd(active, since 2s) 2024-04-22T15:05:52.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:52 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/659289945' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:54.510 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T15:05:54.510 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:05:54.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:54 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:54.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:54 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:54.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:54 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi119", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:05:55.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:55 smithi119 ceph-mon[21835]: Updating smithi119:/etc/ceph/ceph.conf 2024-04-22T15:05:55.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:55 smithi119 ceph-mon[21835]: Updating smithi119:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:05:55.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:55 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:56.930 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T15:05:56.930 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":1,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:03:50.504762Z","created":"2024-04-22T15:03:50.504762Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi119","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:3300","nonce":0},{"type":"v1","addr":"172.21.15.119:6789","nonce":0}]},"addr":"172.21.15.119:6789/0","public_addr":"172.21.15.119:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T15:05:56.932 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 1 2024-04-22T15:05:56.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:56.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:56.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi186", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:05:56.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:05:56.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi186", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:05:56.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi186", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-22T15:05:56.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:56.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:56 smithi119 ceph-mon[21835]: Deploying daemon crash.smithi186 on smithi186 2024-04-22T15:05:57.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:05:57 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/660293892' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:59.289 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T15:05:59.289 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:06:00.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:00.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi186.vtjgwg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:06:00.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi186.vtjgwg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-22T15:06:00.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:06:00.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:00.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:00 smithi119 ceph-mon[21835]: Deploying daemon mgr.smithi186.vtjgwg on smithi186 2024-04-22T15:06:01.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:01 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:02.090 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T15:06:02.090 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":1,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:03:50.504762Z","created":"2024-04-22T15:03:50.504762Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi119","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:3300","nonce":0},{"type":"v1","addr":"172.21.15.119:6789","nonce":0}]},"addr":"172.21.15.119:6789/0","public_addr":"172.21.15.119:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T15:06:02.092 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 1 2024-04-22T15:06:02.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:02 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/3275799597' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:02.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:02.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:06:02.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:03.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:03 smithi119 ceph-mon[21835]: Deploying daemon mon.smithi186 on smithi186 2024-04-22T15:06:03.891 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T15:06:03.891 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:06:06.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:06 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:06.796 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: mon.smithi186@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-04-22T15:06:07.066 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: mon.smithi186@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-22T15:06:07.066 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: mon.smithi186@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:06:07.066 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: mon.smithi186@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:06:07.066 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: mon.smithi186@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:06:07.066 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.066 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:06:07.066 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.066 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.066 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi119", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:06:07.066 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi119", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: Deploying daemon crash.smithi119 on smithi119 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2556227339' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2556227339' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.067 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: Deploying daemon grafana.smithi119 on smithi119 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi119:172.21.15.119=smithi119;smithi186:172.21.15.186=smithi186", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: Saving service mon spec with placement smithi119:172.21.15.119=smithi119;smithi186:172.21.15.186=smithi186;count:2 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/316883395' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/3769942731' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/821602722' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/1640141278' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:07.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: Deploying daemon node-exporter.smithi119 on smithi119 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/1163516188' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14160 172.21.15.119:0/2699496062' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: mgrmap e14: smithi119.jjbxxd(active, since 48s) 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/4155918606' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/4149242112' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: Active manager daemon smithi119.jjbxxd restarted 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: Activating manager daemon smithi119.jjbxxd 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: osdmap e5: 0 total, 0 up, 0 in 2024-04-22T15:06:07.069 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: mgrmap e15: smithi119.jjbxxd(active, starting, since 0.387435s) 2024-04-22T15:06:07.070 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:06:07.070 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi119.jjbxxd", "id": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:06:07.070 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:06:07.070 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:06:07.070 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:06:07.070 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: Manager daemon smithi119.jjbxxd is now available 2024-04-22T15:06:07.070 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: [22/Apr/2024:15:05:50] ENGINE Bus STARTING 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: mgrmap e16: smithi119.jjbxxd(active, since 1.39028s) 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: [22/Apr/2024:15:05:50] ENGINE Serving on https://172.21.15.119:7150 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: [22/Apr/2024:15:05:50] ENGINE Bus STARTED 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.071 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: mgrmap e17: smithi119.jjbxxd(active, since 2s) 2024-04-22T15:06:07.072 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/659289945' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:07.072 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.072 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.072 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi119", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:06:07.072 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: Updating smithi119:/etc/ceph/ceph.conf 2024-04-22T15:06:07.072 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: Updating smithi119:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:06:07.072 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.072 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.072 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.072 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi186", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:06:07.072 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.072 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi186", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi186", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: Deploying daemon crash.smithi186 on smithi186 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/660293892' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi186.vtjgwg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi186.vtjgwg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: Deploying daemon mgr.smithi186.vtjgwg on smithi186 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/3275799597' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:06:07.073 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:07.074 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: Deploying daemon mon.smithi186 on smithi186 2024-04-22T15:06:07.074 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:07.074 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: mon.smithi186@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-04-22T15:06:07.074 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 ceph-mon[24041]: mon.smithi186@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-22T15:06:07.074 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 bash[23991]: a804e52c2c5e51a31e5bae05bff2dc3f62ac15bc04ca460027b19e77186eccc8 2024-04-22T15:06:07.074 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:06 smithi186 systemd[1]: Started Ceph mon.smithi186 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:06:12.104 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:06:12.104 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: mon.smithi119 calling monitor election 2024-04-22T15:06:12.104 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:12.104 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:12.105 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/2739488981' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/crt"}]: dispatch 2024-04-22T15:06:12.105 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:12.105 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: mon.smithi186 calling monitor election 2024-04-22T15:06:12.105 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:12.105 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:12.105 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:12.105 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:12.105 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: mon.smithi119 is new leader, mons smithi119,smithi186 in quorum (ranks 0,1) 2024-04-22T15:06:12.106 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: monmap e2: 2 mons at {smithi119=[v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0],smithi186=[v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0]} 2024-04-22T15:06:12.106 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: fsmap 2024-04-22T15:06:12.106 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: osdmap e5: 0 total, 0 up, 0 in 2024-04-22T15:06:12.106 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: mgrmap e17: smithi119.jjbxxd(active, since 22s) 2024-04-22T15:06:12.106 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: Standby manager daemon smithi186.vtjgwg started 2024-04-22T15:06:12.106 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/2739488981' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:06:12.106 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: overall HEALTH_OK 2024-04-22T15:06:12.106 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/2739488981' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/key"}]: dispatch 2024-04-22T15:06:12.107 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/2739488981' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:06:12.107 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:12.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:06:12.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: mon.smithi119 calling monitor election 2024-04-22T15:06:12.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:12.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:12.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/2739488981' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/crt"}]: dispatch 2024-04-22T15:06:12.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:12.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: mon.smithi186 calling monitor election 2024-04-22T15:06:12.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:12.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:12.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:12.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:12.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: mon.smithi119 is new leader, mons smithi119,smithi186 in quorum (ranks 0,1) 2024-04-22T15:06:12.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: monmap e2: 2 mons at {smithi119=[v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0],smithi186=[v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0]} 2024-04-22T15:06:12.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: fsmap 2024-04-22T15:06:12.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: osdmap e5: 0 total, 0 up, 0 in 2024-04-22T15:06:12.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: mgrmap e17: smithi119.jjbxxd(active, since 22s) 2024-04-22T15:06:12.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: Standby manager daemon smithi186.vtjgwg started 2024-04-22T15:06:12.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/2739488981' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:06:12.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: overall HEALTH_OK 2024-04-22T15:06:12.244 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/2739488981' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/key"}]: dispatch 2024-04-22T15:06:12.245 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/2739488981' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:06:12.245 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:13.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:12 smithi119 ceph-mon[21835]: Deploying daemon node-exporter.smithi186 on smithi186 2024-04-22T15:06:13.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:12 smithi119 ceph-mon[21835]: mgrmap e18: smithi119.jjbxxd(active, since 22s), standbys: smithi186.vtjgwg 2024-04-22T15:06:13.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:12 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi186.vtjgwg", "id": "smithi186.vtjgwg"}]: dispatch 2024-04-22T15:06:13.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:12 smithi119 ceph-mon[21835]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:13.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:12 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:12 smithi186 ceph-mon[24041]: Deploying daemon node-exporter.smithi186 on smithi186 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:12 smithi186 ceph-mon[24041]: mgrmap e18: smithi119.jjbxxd(active, since 22s), standbys: smithi186.vtjgwg 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:12 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi186.vtjgwg", "id": "smithi186.vtjgwg"}]: dispatch 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:12 smithi186 ceph-mon[24041]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:12 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:06:13.334 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-04-22T15:06:13.335 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":2,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:06:06.798166Z","created":"2024-04-22T15:03:50.504762Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi119","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:3300","nonce":0},{"type":"v1","addr":"172.21.15.119:6789","nonce":0}]},"addr":"172.21.15.119:6789/0","public_addr":"172.21.15.119:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi186","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:3300","nonce":0},{"type":"v1","addr":"172.21.15.186:6789","nonce":0}]},"addr":"172.21.15.186:6789/0","public_addr":"172.21.15.186:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-22T15:06:13.335 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 2 2024-04-22T15:06:14.146 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:13 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/2010887320' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:14.170 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-22T15:06:14.170 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-22T15:06:14.195 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:13 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/2010887320' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:15.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:14 smithi119 ceph-mon[21835]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:15.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:14 smithi186 ceph-mon[24041]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:16.013 INFO:teuthology.orchestra.run.smithi119.stdout:# minimal ceph.conf for 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:06:16.014 INFO:teuthology.orchestra.run.smithi119.stdout:[global] 2024-04-22T15:06:16.014 INFO:teuthology.orchestra.run.smithi119.stdout: fsid = 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:06:16.014 INFO:teuthology.orchestra.run.smithi119.stdout: mon_host = [v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0] [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] 2024-04-22T15:06:16.272 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:16 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:16.272 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:16 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/1039120060' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:16.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:16 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:16.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:16 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/1039120060' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:16.556 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-22T15:06:16.556 DEBUG:teuthology.orchestra.run.smithi119:> set -ex 2024-04-22T15:06:16.556 DEBUG:teuthology.orchestra.run.smithi119:> sudo dd of=/etc/ceph/ceph.conf 2024-04-22T15:06:16.622 DEBUG:teuthology.orchestra.run.smithi119:> set -ex 2024-04-22T15:06:16.622 DEBUG:teuthology.orchestra.run.smithi119:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:06:16.685 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-04-22T15:06:16.686 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd of=/etc/ceph/ceph.conf 2024-04-22T15:06:16.713 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-04-22T15:06:16.713 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:06:16.777 INFO:tasks.cephadm:Deploying OSDs... 2024-04-22T15:06:16.778 DEBUG:teuthology.orchestra.run.smithi119:> set -ex 2024-04-22T15:06:16.778 DEBUG:teuthology.orchestra.run.smithi119:> dd if=/scratch_devs of=/dev/stdout 2024-04-22T15:06:16.794 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-22T15:06:16.794 DEBUG:teuthology.orchestra.run.smithi119:> stat /dev/vg_nvme/lv_1 2024-04-22T15:06:16.848 INFO:teuthology.orchestra.run.smithi119.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-22T15:06:16.848 INFO:teuthology.orchestra.run.smithi119.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:16.849 INFO:teuthology.orchestra.run.smithi119.stdout:Device: 5h/5d Inode: 664 Links: 1 2024-04-22T15:06:16.849 INFO:teuthology.orchestra.run.smithi119.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:16.849 INFO:teuthology.orchestra.run.smithi119.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:16.849 INFO:teuthology.orchestra.run.smithi119.stdout:Access: 2024-04-22 15:05:39.995978591 +0000 2024-04-22T15:06:16.849 INFO:teuthology.orchestra.run.smithi119.stdout:Modify: 2024-04-22 15:02:49.596682263 +0000 2024-04-22T15:06:16.849 INFO:teuthology.orchestra.run.smithi119.stdout:Change: 2024-04-22 15:02:49.596682263 +0000 2024-04-22T15:06:16.849 INFO:teuthology.orchestra.run.smithi119.stdout: Birth: 2024-04-22 15:02:49.596682263 +0000 2024-04-22T15:06:16.849 DEBUG:teuthology.orchestra.run.smithi119:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-22T15:06:16.910 INFO:teuthology.orchestra.run.smithi119.stderr:1+0 records in 2024-04-22T15:06:16.911 INFO:teuthology.orchestra.run.smithi119.stderr:1+0 records out 2024-04-22T15:06:16.911 INFO:teuthology.orchestra.run.smithi119.stderr:512 bytes copied, 0.000192374 s, 2.7 MB/s 2024-04-22T15:06:16.912 DEBUG:teuthology.orchestra.run.smithi119:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-22T15:06:16.967 DEBUG:teuthology.orchestra.run.smithi119:> stat /dev/vg_nvme/lv_2 2024-04-22T15:06:17.023 INFO:teuthology.orchestra.run.smithi119.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-22T15:06:17.023 INFO:teuthology.orchestra.run.smithi119.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:17.023 INFO:teuthology.orchestra.run.smithi119.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-04-22T15:06:17.023 INFO:teuthology.orchestra.run.smithi119.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:17.024 INFO:teuthology.orchestra.run.smithi119.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:17.024 INFO:teuthology.orchestra.run.smithi119.stdout:Access: 2024-04-22 15:05:39.995978591 +0000 2024-04-22T15:06:17.024 INFO:teuthology.orchestra.run.smithi119.stdout:Modify: 2024-04-22 15:02:49.997672197 +0000 2024-04-22T15:06:17.024 INFO:teuthology.orchestra.run.smithi119.stdout:Change: 2024-04-22 15:02:49.997672197 +0000 2024-04-22T15:06:17.024 INFO:teuthology.orchestra.run.smithi119.stdout: Birth: 2024-04-22 15:02:49.997672197 +0000 2024-04-22T15:06:17.024 DEBUG:teuthology.orchestra.run.smithi119:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-22T15:06:17.086 INFO:teuthology.orchestra.run.smithi119.stderr:1+0 records in 2024-04-22T15:06:17.087 INFO:teuthology.orchestra.run.smithi119.stderr:1+0 records out 2024-04-22T15:06:17.087 INFO:teuthology.orchestra.run.smithi119.stderr:512 bytes copied, 0.000207334 s, 2.5 MB/s 2024-04-22T15:06:17.088 DEBUG:teuthology.orchestra.run.smithi119:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-22T15:06:17.143 DEBUG:teuthology.orchestra.run.smithi119:> stat /dev/vg_nvme/lv_3 2024-04-22T15:06:17.198 INFO:teuthology.orchestra.run.smithi119.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-22T15:06:17.198 INFO:teuthology.orchestra.run.smithi119.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:17.198 INFO:teuthology.orchestra.run.smithi119.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-04-22T15:06:17.198 INFO:teuthology.orchestra.run.smithi119.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:17.198 INFO:teuthology.orchestra.run.smithi119.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:17.199 INFO:teuthology.orchestra.run.smithi119.stdout:Access: 2024-04-22 15:05:39.996978567 +0000 2024-04-22T15:06:17.199 INFO:teuthology.orchestra.run.smithi119.stdout:Modify: 2024-04-22 15:02:50.413661755 +0000 2024-04-22T15:06:17.199 INFO:teuthology.orchestra.run.smithi119.stdout:Change: 2024-04-22 15:02:50.413661755 +0000 2024-04-22T15:06:17.199 INFO:teuthology.orchestra.run.smithi119.stdout: Birth: 2024-04-22 15:02:50.413661755 +0000 2024-04-22T15:06:17.199 DEBUG:teuthology.orchestra.run.smithi119:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-22T15:06:17.262 INFO:teuthology.orchestra.run.smithi119.stderr:1+0 records in 2024-04-22T15:06:17.262 INFO:teuthology.orchestra.run.smithi119.stderr:1+0 records out 2024-04-22T15:06:17.262 INFO:teuthology.orchestra.run.smithi119.stderr:512 bytes copied, 0.000213337 s, 2.4 MB/s 2024-04-22T15:06:17.263 DEBUG:teuthology.orchestra.run.smithi119:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-22T15:06:17.321 DEBUG:teuthology.orchestra.run.smithi119:> stat /dev/vg_nvme/lv_4 2024-04-22T15:06:17.375 INFO:teuthology.orchestra.run.smithi119.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-22T15:06:17.375 INFO:teuthology.orchestra.run.smithi119.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:17.375 INFO:teuthology.orchestra.run.smithi119.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-22T15:06:17.375 INFO:teuthology.orchestra.run.smithi119.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:17.375 INFO:teuthology.orchestra.run.smithi119.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:17.375 INFO:teuthology.orchestra.run.smithi119.stdout:Access: 2024-04-22 15:05:39.996978567 +0000 2024-04-22T15:06:17.376 INFO:teuthology.orchestra.run.smithi119.stdout:Modify: 2024-04-22 15:02:51.586632310 +0000 2024-04-22T15:06:17.376 INFO:teuthology.orchestra.run.smithi119.stdout:Change: 2024-04-22 15:02:51.586632310 +0000 2024-04-22T15:06:17.376 INFO:teuthology.orchestra.run.smithi119.stdout: Birth: 2024-04-22 15:02:51.586632310 +0000 2024-04-22T15:06:17.376 DEBUG:teuthology.orchestra.run.smithi119:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-22T15:06:17.437 INFO:teuthology.orchestra.run.smithi119.stderr:1+0 records in 2024-04-22T15:06:17.437 INFO:teuthology.orchestra.run.smithi119.stderr:1+0 records out 2024-04-22T15:06:17.437 INFO:teuthology.orchestra.run.smithi119.stderr:512 bytes copied, 0.000174681 s, 2.9 MB/s 2024-04-22T15:06:17.438 DEBUG:teuthology.orchestra.run.smithi119:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-22T15:06:17.495 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-04-22T15:06:17.495 DEBUG:teuthology.orchestra.run.smithi186:> dd if=/scratch_devs of=/dev/stdout 2024-04-22T15:06:17.511 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-22T15:06:17.512 DEBUG:teuthology.orchestra.run.smithi186:> stat /dev/vg_nvme/lv_1 2024-04-22T15:06:17.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:17 smithi186 ceph-mon[24041]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:17.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:17 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:17.540 INFO:teuthology.orchestra.run.smithi186.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-22T15:06:17.540 INFO:teuthology.orchestra.run.smithi186.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:17.540 INFO:teuthology.orchestra.run.smithi186.stdout:Device: 5h/5d Inode: 664 Links: 1 2024-04-22T15:06:17.540 INFO:teuthology.orchestra.run.smithi186.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:17.540 INFO:teuthology.orchestra.run.smithi186.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:17.540 INFO:teuthology.orchestra.run.smithi186.stdout:Access: 2024-04-22 15:06:12.376386537 +0000 2024-04-22T15:06:17.540 INFO:teuthology.orchestra.run.smithi186.stdout:Modify: 2024-04-22 15:02:53.299584967 +0000 2024-04-22T15:06:17.540 INFO:teuthology.orchestra.run.smithi186.stdout:Change: 2024-04-22 15:02:53.299584967 +0000 2024-04-22T15:06:17.540 INFO:teuthology.orchestra.run.smithi186.stdout: Birth: 2024-04-22 15:02:53.299584967 +0000 2024-04-22T15:06:17.541 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-22T15:06:17.603 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records in 2024-04-22T15:06:17.603 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records out 2024-04-22T15:06:17.603 INFO:teuthology.orchestra.run.smithi186.stderr:512 bytes copied, 0.000286809 s, 1.8 MB/s 2024-04-22T15:06:17.604 DEBUG:teuthology.orchestra.run.smithi186:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-22T15:06:17.662 DEBUG:teuthology.orchestra.run.smithi186:> stat /dev/vg_nvme/lv_2 2024-04-22T15:06:17.717 INFO:teuthology.orchestra.run.smithi186.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-22T15:06:17.717 INFO:teuthology.orchestra.run.smithi186.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:17.717 INFO:teuthology.orchestra.run.smithi186.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-22T15:06:17.717 INFO:teuthology.orchestra.run.smithi186.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:17.717 INFO:teuthology.orchestra.run.smithi186.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:17.718 INFO:teuthology.orchestra.run.smithi186.stdout:Access: 2024-04-22 15:06:12.376386537 +0000 2024-04-22T15:06:17.718 INFO:teuthology.orchestra.run.smithi186.stdout:Modify: 2024-04-22 15:02:53.702577580 +0000 2024-04-22T15:06:17.718 INFO:teuthology.orchestra.run.smithi186.stdout:Change: 2024-04-22 15:02:53.702577580 +0000 2024-04-22T15:06:17.718 INFO:teuthology.orchestra.run.smithi186.stdout: Birth: 2024-04-22 15:02:53.702577580 +0000 2024-04-22T15:06:17.718 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-22T15:06:17.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:17 smithi119 ceph-mon[21835]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:17.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:17 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:17.779 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records in 2024-04-22T15:06:17.780 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records out 2024-04-22T15:06:17.780 INFO:teuthology.orchestra.run.smithi186.stderr:512 bytes copied, 0.000172869 s, 3.0 MB/s 2024-04-22T15:06:17.781 DEBUG:teuthology.orchestra.run.smithi186:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-22T15:06:17.837 DEBUG:teuthology.orchestra.run.smithi186:> stat /dev/vg_nvme/lv_3 2024-04-22T15:06:17.892 INFO:teuthology.orchestra.run.smithi186.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-22T15:06:17.892 INFO:teuthology.orchestra.run.smithi186.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:17.893 INFO:teuthology.orchestra.run.smithi186.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-22T15:06:17.893 INFO:teuthology.orchestra.run.smithi186.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:17.893 INFO:teuthology.orchestra.run.smithi186.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:17.893 INFO:teuthology.orchestra.run.smithi186.stdout:Access: 2024-04-22 15:06:12.377386520 +0000 2024-04-22T15:06:17.893 INFO:teuthology.orchestra.run.smithi186.stdout:Modify: 2024-04-22 15:02:54.214568194 +0000 2024-04-22T15:06:17.893 INFO:teuthology.orchestra.run.smithi186.stdout:Change: 2024-04-22 15:02:54.214568194 +0000 2024-04-22T15:06:17.893 INFO:teuthology.orchestra.run.smithi186.stdout: Birth: 2024-04-22 15:02:54.214568194 +0000 2024-04-22T15:06:17.893 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-22T15:06:17.954 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records in 2024-04-22T15:06:17.955 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records out 2024-04-22T15:06:17.955 INFO:teuthology.orchestra.run.smithi186.stderr:512 bytes copied, 0.000194 s, 2.6 MB/s 2024-04-22T15:06:17.956 DEBUG:teuthology.orchestra.run.smithi186:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-22T15:06:18.011 DEBUG:teuthology.orchestra.run.smithi186:> stat /dev/vg_nvme/lv_4 2024-04-22T15:06:18.066 INFO:teuthology.orchestra.run.smithi186.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-22T15:06:18.066 INFO:teuthology.orchestra.run.smithi186.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:18.066 INFO:teuthology.orchestra.run.smithi186.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-22T15:06:18.066 INFO:teuthology.orchestra.run.smithi186.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:18.066 INFO:teuthology.orchestra.run.smithi186.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:18.066 INFO:teuthology.orchestra.run.smithi186.stdout:Access: 2024-04-22 15:06:12.377386520 +0000 2024-04-22T15:06:18.067 INFO:teuthology.orchestra.run.smithi186.stdout:Modify: 2024-04-22 15:02:54.618560788 +0000 2024-04-22T15:06:18.067 INFO:teuthology.orchestra.run.smithi186.stdout:Change: 2024-04-22 15:02:54.618560788 +0000 2024-04-22T15:06:18.067 INFO:teuthology.orchestra.run.smithi186.stdout: Birth: 2024-04-22 15:02:54.618560788 +0000 2024-04-22T15:06:18.067 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-22T15:06:18.128 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records in 2024-04-22T15:06:18.128 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records out 2024-04-22T15:06:18.128 INFO:teuthology.orchestra.run.smithi186.stderr:512 bytes copied, 0.000190144 s, 2.7 MB/s 2024-04-22T15:06:18.129 DEBUG:teuthology.orchestra.run.smithi186:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-22T15:06:18.185 INFO:tasks.cephadm:Deploying osd.0 on smithi119 with /dev/vg_nvme/lv_4... 2024-04-22T15:06:18.185 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-22T15:06:18.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:18 smithi119 ceph-mon[21835]: Deploying daemon prometheus.smithi119 on smithi119 2024-04-22T15:06:18.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:18 smithi186 ceph-mon[24041]: Deploying daemon prometheus.smithi119 on smithi119 2024-04-22T15:06:19.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:19 smithi119 ceph-mon[21835]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:19.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:19 smithi186 ceph-mon[24041]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:21.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:21 smithi119 ceph-mon[21835]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:21.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:21 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:21.335 INFO:teuthology.orchestra.run.smithi119.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-22T15:06:21.336 INFO:teuthology.orchestra.run.smithi119.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-22T15:06:21.336 INFO:teuthology.orchestra.run.smithi119.stderr: stderr: 10+0 records in 2024-04-22T15:06:21.336 INFO:teuthology.orchestra.run.smithi119.stderr:10+0 records out 2024-04-22T15:06:21.336 INFO:teuthology.orchestra.run.smithi119.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0134843 s, 778 MB/s 2024-04-22T15:06:21.336 INFO:teuthology.orchestra.run.smithi119.stderr:--> Zapping successful for: 2024-04-22T15:06:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:21 smithi186 ceph-mon[24041]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:21 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:22.261 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi119:vg_nvme/lv_4 2024-04-22T15:06:23.730 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:23 smithi119 ceph-mon[21835]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:23.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:23 smithi186 ceph-mon[24041]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:24.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:24 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:06:24.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:24 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:06:24.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:24 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:24.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:24 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:06:24.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:24 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:06:24.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:24 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:25.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:25 smithi119 ceph-mon[21835]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:25.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:25 smithi119 ceph-mon[21835]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi119:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:06:25.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:25 smithi186 ceph-mon[24041]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:25.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:25 smithi186 ceph-mon[24041]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi119:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:06:27.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:27 smithi119 ceph-mon[21835]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:27.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:27 smithi186 ceph-mon[24041]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:29.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:29 smithi119 ceph-mon[21835]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:29.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:29 smithi186 ceph-mon[24041]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:31.739 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:31 smithi119 ceph-mon[21835]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:31.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:31 smithi186 ceph-mon[24041]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:33.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:32 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:33.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:32 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:33.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:32 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:33.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:32 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:33.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:32 smithi119 ceph-mon[21835]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:32 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:32 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:32 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:32 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:32 smithi186 ceph-mon[24041]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:34 smithi119 ceph-mon[21835]: Updating smithi186:/etc/ceph/ceph.conf 2024-04-22T15:06:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:34 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/4034540078' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5e297b79-9f0a-4ca7-b0c1-bbb13a534936"}]: dispatch 2024-04-22T15:06:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:34 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/4034540078' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5e297b79-9f0a-4ca7-b0c1-bbb13a534936"}]': finished 2024-04-22T15:06:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:34 smithi119 ceph-mon[21835]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T15:06:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:35.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:35.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:34 smithi186 ceph-mon[24041]: Updating smithi186:/etc/ceph/ceph.conf 2024-04-22T15:06:35.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:35.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:34 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/4034540078' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5e297b79-9f0a-4ca7-b0c1-bbb13a534936"}]: dispatch 2024-04-22T15:06:35.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:34 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/4034540078' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5e297b79-9f0a-4ca7-b0c1-bbb13a534936"}]': finished 2024-04-22T15:06:35.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:34 smithi186 ceph-mon[24041]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T15:06:35.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:35.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:35 smithi119 ceph-mon[21835]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:06:35.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:35 smithi119 ceph-mon[21835]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:35.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:35.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:35 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3217405483' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:06:36.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:35 smithi186 ceph-mon[24041]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:06:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:35 smithi186 ceph-mon[24041]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:35 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/3217405483' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:06:37.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:37 smithi119 ceph-mon[21835]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:37.923 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:38.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:37 smithi186 ceph-mon[24041]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:38.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:38.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:38 smithi119 ceph-mon[21835]: Updating smithi119:/etc/ceph/ceph.conf 2024-04-22T15:06:38.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:38 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:38.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:38 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:38.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:38 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:06:38.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:38 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:06:38.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:38 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:39.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:38 smithi186 ceph-mon[24041]: Updating smithi119:/etc/ceph/ceph.conf 2024-04-22T15:06:39.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:38 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:39.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:38 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:39.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:38 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:06:39.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:38 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:06:39.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:38 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:39 smithi119 ceph-mon[21835]: Reconfiguring mon.smithi119 (unknown last config time)... 2024-04-22T15:06:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:39 smithi119 ceph-mon[21835]: Reconfiguring daemon mon.smithi119 on smithi119 2024-04-22T15:06:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:39 smithi119 ceph-mon[21835]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:39 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:39 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi119.jjbxxd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:06:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:39 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:06:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:39 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:39 smithi186 ceph-mon[24041]: Reconfiguring mon.smithi119 (unknown last config time)... 2024-04-22T15:06:40.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:39 smithi186 ceph-mon[24041]: Reconfiguring daemon mon.smithi119 on smithi119 2024-04-22T15:06:40.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:39 smithi186 ceph-mon[24041]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:40.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:39 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:40.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:39 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi119.jjbxxd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:06:40.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:39 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:06:40.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:39 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:40.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:40 smithi119 ceph-mon[21835]: Reconfiguring mgr.smithi119.jjbxxd (unknown last config time)... 2024-04-22T15:06:40.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:40 smithi119 ceph-mon[21835]: Reconfiguring daemon mgr.smithi119.jjbxxd on smithi119 2024-04-22T15:06:41.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:40 smithi186 ceph-mon[24041]: Reconfiguring mgr.smithi119.jjbxxd (unknown last config time)... 2024-04-22T15:06:41.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:40 smithi186 ceph-mon[24041]: Reconfiguring daemon mgr.smithi119.jjbxxd on smithi119 2024-04-22T15:06:41.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:41 smithi119 ceph-mon[21835]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:41.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:41 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:42.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:41 smithi186 ceph-mon[24041]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:42.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:41 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:42.874 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:42 smithi119 ceph-mon[21835]: Reconfiguring alertmanager.smithi119 (dependencies changed)... 2024-04-22T15:06:42.874 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:42 smithi119 ceph-mon[21835]: Reconfiguring daemon alertmanager.smithi119 on smithi119 2024-04-22T15:06:43.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:42 smithi186 ceph-mon[24041]: Reconfiguring alertmanager.smithi119 (dependencies changed)... 2024-04-22T15:06:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:42 smithi186 ceph-mon[24041]: Reconfiguring daemon alertmanager.smithi119 on smithi119 2024-04-22T15:06:43.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:43 smithi119 ceph-mon[21835]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:43.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:43 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:06:43.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:43 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:44.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:43 smithi186 ceph-mon[24041]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:44.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:43 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:06:44.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:43 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:44.961 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:44 smithi119 ceph-mon[21835]: Deploying daemon osd.0 on smithi119 2024-04-22T15:06:45.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:44 smithi186 ceph-mon[24041]: Deploying daemon osd.0 on smithi119 2024-04-22T15:06:45.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:45 smithi186 ceph-mon[24041]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:45 smithi119 ceph-mon[21835]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:47.937 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:47 smithi119 ceph-mon[21835]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:47.937 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:47 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:47.937 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:47 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi119", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:06:47.937 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:47 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:48.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:47 smithi186 ceph-mon[24041]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:48.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:47 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:48.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:47 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi119", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:06:48.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:47 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:48.949 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:48 smithi119 ceph-mon[21835]: Reconfiguring crash.smithi119 (monmap changed)... 2024-04-22T15:06:48.949 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:48 smithi119 ceph-mon[21835]: Reconfiguring daemon crash.smithi119 on smithi119 2024-04-22T15:06:49.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:48 smithi186 ceph-mon[24041]: Reconfiguring crash.smithi119 (monmap changed)... 2024-04-22T15:06:49.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:48 smithi186 ceph-mon[24041]: Reconfiguring daemon crash.smithi119 on smithi119 2024-04-22T15:06:49.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:49 smithi119 ceph-mon[21835]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:49 smithi186 ceph-mon[24041]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:50.807 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:06:50.807 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:06:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:06:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:06:51.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:51 smithi119 ceph-mon[21835]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:51 smithi186 ceph-mon[24041]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:53.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:53 smithi119 ceph-mon[21835]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:53.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:53 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:54.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:53 smithi186 ceph-mon[24041]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:54.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:53 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:55.102 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:54 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:55.102 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:54 smithi119 ceph-mon[21835]: Reconfiguring grafana.smithi119 (dependencies changed)... 2024-04-22T15:06:55.102 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:54 smithi119 ceph-mon[21835]: Reconfiguring daemon grafana.smithi119 on smithi119 2024-04-22T15:06:55.103 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:54 smithi119 ceph-mon[21835]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:55.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:54 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:55.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:54 smithi186 ceph-mon[24041]: Reconfiguring grafana.smithi119 (dependencies changed)... 2024-04-22T15:06:55.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:54 smithi186 ceph-mon[24041]: Reconfiguring daemon grafana.smithi119 on smithi119 2024-04-22T15:06:55.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:54 smithi186 ceph-mon[24041]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:55.930 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:55 smithi119 ceph-mon[21835]: from='osd.0 [v2:172.21.15.119:6802/1289753238,v1:172.21.15.119:6803/1289753238]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T15:06:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:55 smithi186 ceph-mon[24041]: from='osd.0 [v2:172.21.15.119:6802/1289753238,v1:172.21.15.119:6803/1289753238]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T15:06:57.230 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:56 smithi119 ceph-mon[21835]: from='osd.0 [v2:172.21.15.119:6802/1289753238,v1:172.21.15.119:6803/1289753238]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T15:06:57.231 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:56 smithi119 ceph-mon[21835]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T15:06:57.231 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:57.231 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:56 smithi119 ceph-mon[21835]: from='osd.0 [v2:172.21.15.119:6802/1289753238,v1:172.21.15.119:6803/1289753238]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:06:57.231 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:56 smithi119 ceph-mon[21835]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:57.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:56 smithi186 ceph-mon[24041]: from='osd.0 [v2:172.21.15.119:6802/1289753238,v1:172.21.15.119:6803/1289753238]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T15:06:57.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:56 smithi186 ceph-mon[24041]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T15:06:57.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:57.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:56 smithi186 ceph-mon[24041]: from='osd.0 [v2:172.21.15.119:6802/1289753238,v1:172.21.15.119:6803/1289753238]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:06:57.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:56 smithi186 ceph-mon[24041]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:58.102 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:57 smithi119 ceph-mon[21835]: from='osd.0 [v2:172.21.15.119:6802/1289753238,v1:172.21.15.119:6803/1289753238]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]': finished 2024-04-22T15:06:58.102 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:57 smithi119 ceph-mon[21835]: osdmap e8: 1 total, 0 up, 1 in 2024-04-22T15:06:58.102 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:57 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:58.102 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:57 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:57 smithi186 ceph-mon[24041]: from='osd.0 [v2:172.21.15.119:6802/1289753238,v1:172.21.15.119:6803/1289753238]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]': finished 2024-04-22T15:06:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:57 smithi186 ceph-mon[24041]: osdmap e8: 1 total, 0 up, 1 in 2024-04-22T15:06:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:57 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:57 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:58.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:58 smithi119 ceph-mon[21835]: purged_snaps scrub starts 2024-04-22T15:06:58.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:58 smithi119 ceph-mon[21835]: purged_snaps scrub ok 2024-04-22T15:06:58.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:58 smithi119 ceph-mon[21835]: osd.0 [v2:172.21.15.119:6802/1289753238,v1:172.21.15.119:6803/1289753238] boot 2024-04-22T15:06:58.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:58 smithi119 ceph-mon[21835]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T15:06:58.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:58 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:58.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:58 smithi119 ceph-mon[21835]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:58.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:58 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:58.984 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:58 smithi119 ceph-mon[21835]: Reconfiguring crash.smithi186 (monmap changed)... 2024-04-22T15:06:58.984 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:58 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi186", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:06:58.984 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:58 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:58.984 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:06:58 smithi119 ceph-mon[21835]: Reconfiguring daemon crash.smithi186 on smithi186 2024-04-22T15:06:59.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:58 smithi186 ceph-mon[24041]: purged_snaps scrub starts 2024-04-22T15:06:59.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:58 smithi186 ceph-mon[24041]: purged_snaps scrub ok 2024-04-22T15:06:59.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:58 smithi186 ceph-mon[24041]: osd.0 [v2:172.21.15.119:6802/1289753238,v1:172.21.15.119:6803/1289753238] boot 2024-04-22T15:06:59.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:58 smithi186 ceph-mon[24041]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T15:06:59.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:58 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:59.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:58 smithi186 ceph-mon[24041]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:59.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:58 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:06:59.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:58 smithi186 ceph-mon[24041]: Reconfiguring crash.smithi186 (monmap changed)... 2024-04-22T15:06:59.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:58 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi186", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:06:59.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:58 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:59.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:06:58 smithi186 ceph-mon[24041]: Reconfiguring daemon crash.smithi186 on smithi186 2024-04-22T15:07:00.253 INFO:teuthology.orchestra.run.smithi119.stdout:Created osd(s) 0 on host 'smithi119' 2024-04-22T15:07:00.817 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:00 smithi119 ceph-mon[21835]: osdmap e10: 1 total, 1 up, 1 in 2024-04-22T15:07:00.817 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:00.817 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi186.vtjgwg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:07:00.817 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:07:00.817 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:00.817 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:00.844 DEBUG:teuthology.orchestra.run.smithi119:osd.0> sudo journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.0.service 2024-04-22T15:07:00.847 INFO:tasks.cephadm:Deploying osd.1 on smithi119 with /dev/vg_nvme/lv_3... 2024-04-22T15:07:00.847 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-22T15:07:01.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:00 smithi186 ceph-mon[24041]: osdmap e10: 1 total, 1 up, 1 in 2024-04-22T15:07:01.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:01.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi186.vtjgwg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:07:01.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:07:01.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:01.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:01.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:01 smithi119 ceph-mon[21835]: Reconfiguring mgr.smithi186.vtjgwg (monmap changed)... 2024-04-22T15:07:01.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:01 smithi119 ceph-mon[21835]: Reconfiguring daemon mgr.smithi186.vtjgwg on smithi186 2024-04-22T15:07:01.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:01 smithi119 ceph-mon[21835]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:01.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:01 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:01.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:01 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:07:01.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:01 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:07:01.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:01 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:01.993 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:01 smithi186 ceph-mon[24041]: Reconfiguring mgr.smithi186.vtjgwg (monmap changed)... 2024-04-22T15:07:01.994 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:01 smithi186 ceph-mon[24041]: Reconfiguring daemon mgr.smithi186.vtjgwg on smithi186 2024-04-22T15:07:01.994 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:01 smithi186 ceph-mon[24041]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:01.994 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:01 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:01.994 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:01 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:07:01.994 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:01 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:07:01.994 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:01 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:02.797 INFO:teuthology.orchestra.run.smithi119.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-22T15:07:02.797 INFO:teuthology.orchestra.run.smithi119.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-22T15:07:02.797 INFO:teuthology.orchestra.run.smithi119.stderr: stderr: 10+0 records in 2024-04-22T15:07:02.797 INFO:teuthology.orchestra.run.smithi119.stderr:10+0 records out 2024-04-22T15:07:02.797 INFO:teuthology.orchestra.run.smithi119.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0151339 s, 693 MB/s 2024-04-22T15:07:02.797 INFO:teuthology.orchestra.run.smithi119.stderr:--> Zapping successful for: 2024-04-22T15:07:02.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: Reconfiguring mon.smithi186 (monmap changed)... 2024-04-22T15:07:02.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: Reconfiguring daemon mon.smithi186 on smithi186 2024-04-22T15:07:02.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:02.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:07:02.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.119:9093"}]: dispatch 2024-04-22T15:07:02.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:02.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:07:02.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.119:3000"}]: dispatch 2024-04-22T15:07:02.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:02.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:07:02.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.119:9095"}]: dispatch 2024-04-22T15:07:02.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:02.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:02.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:02.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:02 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:03.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: Reconfiguring mon.smithi186 (monmap changed)... 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: Reconfiguring daemon mon.smithi186 on smithi186 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.119:9093"}]: dispatch 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.119:3000"}]: dispatch 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.119:9095"}]: dispatch 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:03.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:03.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:02 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:03.486 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi119:vg_nvme/lv_3 2024-04-22T15:07:03.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:03 smithi119 ceph-mon[21835]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:03.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:03 smithi119 ceph-mon[21835]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:07:03.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:03 smithi119 ceph-mon[21835]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.119:9093"}]: dispatch 2024-04-22T15:07:03.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:03 smithi119 ceph-mon[21835]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:07:03.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:03 smithi119 ceph-mon[21835]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.119:3000"}]: dispatch 2024-04-22T15:07:03.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:03 smithi119 ceph-mon[21835]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:07:03.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:03 smithi119 ceph-mon[21835]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.119:9095"}]: dispatch 2024-04-22T15:07:04.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:03 smithi186 ceph-mon[24041]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:04.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:03 smithi186 ceph-mon[24041]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:07:04.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:03 smithi186 ceph-mon[24041]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.119:9093"}]: dispatch 2024-04-22T15:07:04.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:03 smithi186 ceph-mon[24041]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:07:04.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:03 smithi186 ceph-mon[24041]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.119:3000"}]: dispatch 2024-04-22T15:07:04.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:03 smithi186 ceph-mon[24041]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:07:04.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:03 smithi186 ceph-mon[24041]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.119:9095"}]: dispatch 2024-04-22T15:07:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:05 smithi186 ceph-mon[24041]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:05.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:05 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:05.621 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:05 smithi119 ceph-mon[21835]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:05.622 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:05 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:06.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:06 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:06.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:06 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:06.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:06 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:06.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:07.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:07 smithi119 ceph-mon[21835]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi119:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:07.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:07 smithi119 ceph-mon[21835]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:07.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:07 smithi186 ceph-mon[24041]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi119:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:07.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:07 smithi186 ceph-mon[24041]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:09.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:09 smithi119 ceph-mon[21835]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:09.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:09 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/1153146414' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95c40890-6713-4e2a-94f3-9a2adb44dc20"}]: dispatch 2024-04-22T15:07:09.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:09 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/1153146414' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "95c40890-6713-4e2a-94f3-9a2adb44dc20"}]': finished 2024-04-22T15:07:09.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:09 smithi119 ceph-mon[21835]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T15:07:09.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:09 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:09.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:09 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/1139209646' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:07:09.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:09 smithi186 ceph-mon[24041]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:09.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:09 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/1153146414' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95c40890-6713-4e2a-94f3-9a2adb44dc20"}]: dispatch 2024-04-22T15:07:09.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:09 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/1153146414' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "95c40890-6713-4e2a-94f3-9a2adb44dc20"}]': finished 2024-04-22T15:07:09.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:09 smithi186 ceph-mon[24041]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T15:07:09.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:09 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:09.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:09 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/1139209646' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:07:11.409 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:11 smithi119 ceph-mon[21835]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:11.409 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:11.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:11 smithi186 ceph-mon[24041]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:11.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:13.702 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:13 smithi119 ceph-mon[21835]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:13 smithi186 ceph-mon[24041]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:15.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:15 smithi186 ceph-mon[24041]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:15.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:15 smithi186 ceph-mon[24041]: Detected new or changed devices on smithi119 2024-04-22T15:07:15.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:15.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:15.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:15 smithi186 ceph-mon[24041]: Adjusting osd_memory_target on smithi119 to 12864M 2024-04-22T15:07:15.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:15.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:15.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:15.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:15.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:15.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:15 smithi119 ceph-mon[21835]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:15.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:15 smithi119 ceph-mon[21835]: Detected new or changed devices on smithi119 2024-04-22T15:07:15.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:15.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:15.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:15 smithi119 ceph-mon[21835]: Adjusting osd_memory_target on smithi119 to 12864M 2024-04-22T15:07:15.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:15.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:15.625 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:15.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:15.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:17.232 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:17 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:17.232 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:17 smithi119 ceph-mon[21835]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:17.232 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:17 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:07:17.232 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:17 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:17.233 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:17 smithi119 ceph-mon[21835]: Deploying daemon osd.1 on smithi119 2024-04-22T15:07:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:17 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:17.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:17 smithi186 ceph-mon[24041]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:17.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:17 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:07:17.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:17 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:17.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:17 smithi186 ceph-mon[24041]: Deploying daemon osd.1 on smithi119 2024-04-22T15:07:19.183 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:19 smithi119 ceph-mon[21835]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:19.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:19 smithi186 ceph-mon[24041]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:21.464 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:21 smithi119 ceph-mon[21835]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:21 smithi186 ceph-mon[24041]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:23.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:22 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:23.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:22 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:23.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:22 smithi119 ceph-mon[21835]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:22 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:22 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:23.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:22 smithi186 ceph-mon[24041]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:24.955 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:24 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:24.955 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:24 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:24.956 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:24 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:24.956 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:24 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:24.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:24 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:24.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:24 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:24.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:24 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:24.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:24 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:25.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:25 smithi119 ceph-mon[21835]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:25.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:25 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:25 smithi186 ceph-mon[24041]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:26.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:25 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:27.121 INFO:teuthology.orchestra.run.smithi119.stdout:Created osd(s) 1 on host 'smithi119' 2024-04-22T15:07:27.962 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:27 smithi119 ceph-mon[21835]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:27.962 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:27 smithi119 ceph-mon[21835]: from='osd.1 [v2:172.21.15.119:6810/141206705,v1:172.21.15.119:6811/141206705]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:07:27.962 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:27.986 DEBUG:teuthology.orchestra.run.smithi119:osd.1> sudo journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.1.service 2024-04-22T15:07:27.989 INFO:tasks.cephadm:Deploying osd.2 on smithi119 with /dev/vg_nvme/lv_2... 2024-04-22T15:07:27.989 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-22T15:07:28.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:27 smithi186 ceph-mon[24041]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:28.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:27 smithi186 ceph-mon[24041]: from='osd.1 [v2:172.21.15.119:6810/141206705,v1:172.21.15.119:6811/141206705]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:07:28.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:28 smithi119 ceph-mon[21835]: from='osd.1 [v2:172.21.15.119:6810/141206705,v1:172.21.15.119:6811/141206705]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T15:07:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:28 smithi119 ceph-mon[21835]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T15:07:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:28 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:28 smithi119 ceph-mon[21835]: from='osd.1 [v2:172.21.15.119:6810/141206705,v1:172.21.15.119:6811/141206705]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:07:28.992 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:07:28 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[36299]: 2024-04-22T15:07:28.692+0000 7fd51ca6d700 -1 osd.1 0 waiting for initial osdmap 2024-04-22T15:07:28.993 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:07:28 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[36299]: 2024-04-22T15:07:28.695+0000 7fd517404700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:07:29.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:28 smithi186 ceph-mon[24041]: from='osd.1 [v2:172.21.15.119:6810/141206705,v1:172.21.15.119:6811/141206705]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T15:07:29.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:28 smithi186 ceph-mon[24041]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T15:07:29.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:28 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:29.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:28 smithi186 ceph-mon[24041]: from='osd.1 [v2:172.21.15.119:6810/141206705,v1:172.21.15.119:6811/141206705]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:07:29.794 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:29 smithi119 ceph-mon[21835]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:29.794 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:29 smithi119 ceph-mon[21835]: from='osd.1 [v2:172.21.15.119:6810/141206705,v1:172.21.15.119:6811/141206705]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]': finished 2024-04-22T15:07:29.795 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:29 smithi119 ceph-mon[21835]: osdmap e13: 2 total, 1 up, 2 in 2024-04-22T15:07:29.795 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:29 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:29.795 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:29 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:29 smithi186 ceph-mon[24041]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:30.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:29 smithi186 ceph-mon[24041]: from='osd.1 [v2:172.21.15.119:6810/141206705,v1:172.21.15.119:6811/141206705]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]': finished 2024-04-22T15:07:30.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:29 smithi186 ceph-mon[24041]: osdmap e13: 2 total, 1 up, 2 in 2024-04-22T15:07:30.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:29 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:30.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:29 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:30.818 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:30 smithi119 ceph-mon[21835]: purged_snaps scrub starts 2024-04-22T15:07:30.818 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:30 smithi119 ceph-mon[21835]: purged_snaps scrub ok 2024-04-22T15:07:30.818 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:30 smithi119 ceph-mon[21835]: osd.1 [v2:172.21.15.119:6810/141206705,v1:172.21.15.119:6811/141206705] boot 2024-04-22T15:07:30.818 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:30 smithi119 ceph-mon[21835]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T15:07:30.818 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:30 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:31.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:30 smithi186 ceph-mon[24041]: purged_snaps scrub starts 2024-04-22T15:07:31.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:30 smithi186 ceph-mon[24041]: purged_snaps scrub ok 2024-04-22T15:07:31.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:30 smithi186 ceph-mon[24041]: osd.1 [v2:172.21.15.119:6810/141206705,v1:172.21.15.119:6811/141206705] boot 2024-04-22T15:07:31.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:30 smithi186 ceph-mon[24041]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T15:07:31.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:30 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:31.185 INFO:teuthology.orchestra.run.smithi119.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-22T15:07:31.185 INFO:teuthology.orchestra.run.smithi119.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-22T15:07:31.186 INFO:teuthology.orchestra.run.smithi119.stderr: stderr: 10+0 records in 2024-04-22T15:07:31.186 INFO:teuthology.orchestra.run.smithi119.stderr:10+0 records out 2024-04-22T15:07:31.186 INFO:teuthology.orchestra.run.smithi119.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0139004 s, 754 MB/s 2024-04-22T15:07:31.186 INFO:teuthology.orchestra.run.smithi119.stderr:--> Zapping successful for: 2024-04-22T15:07:31.967 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:31 smithi119 ceph-mon[21835]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:31.967 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:31 smithi119 ceph-mon[21835]: osdmap e15: 2 total, 2 up, 2 in 2024-04-22T15:07:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:31 smithi186 ceph-mon[24041]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:31 smithi186 ceph-mon[24041]: osdmap e15: 2 total, 2 up, 2 in 2024-04-22T15:07:32.253 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi119:vg_nvme/lv_2 2024-04-22T15:07:33.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:33 smithi119 ceph-mon[21835]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:33.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:33 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:33.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:33 smithi186 ceph-mon[24041]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:33.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:33 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:34.687 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:34.687 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:34.687 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:34.735 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:34.736 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:34.736 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:35.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:35 smithi119 ceph-mon[21835]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:35.636 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:35 smithi119 ceph-mon[21835]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi119:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:35.669 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:35 smithi186 ceph-mon[24041]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:35.669 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:35 smithi186 ceph-mon[24041]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi119:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:37.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:37 smithi119 ceph-mon[21835]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:37.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:37.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:37.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:37.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:37.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:37.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:37.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:37.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:37 smithi186 ceph-mon[24041]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:37.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:37.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:37.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:37.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:37.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:37.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:38.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:38 smithi119 ceph-mon[21835]: Adjusting osd_memory_target on smithi119 to 6432M 2024-04-22T15:07:38.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:38 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2334890325' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "36c6bc11-2dd0-48f5-b284-73b17aaeea69"}]: dispatch 2024-04-22T15:07:38.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:38 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2334890325' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "36c6bc11-2dd0-48f5-b284-73b17aaeea69"}]': finished 2024-04-22T15:07:38.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:38 smithi119 ceph-mon[21835]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T15:07:38.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:38 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:38.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:38 smithi186 ceph-mon[24041]: Adjusting osd_memory_target on smithi119 to 6432M 2024-04-22T15:07:38.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:38 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2334890325' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "36c6bc11-2dd0-48f5-b284-73b17aaeea69"}]: dispatch 2024-04-22T15:07:38.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:38 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2334890325' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "36c6bc11-2dd0-48f5-b284-73b17aaeea69"}]': finished 2024-04-22T15:07:38.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:38 smithi186 ceph-mon[24041]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T15:07:38.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:38 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:39.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:39 smithi119 ceph-mon[21835]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:39.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:39 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/1663914988' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:07:39.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:39 smithi186 ceph-mon[24041]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:39.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:39 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/1663914988' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:07:41.723 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:41 smithi119 ceph-mon[21835]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:41.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:41 smithi186 ceph-mon[24041]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:43.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:43 smithi119 ceph-mon[21835]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:43.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:43 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:43.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:43 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:43.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:43 smithi186 ceph-mon[24041]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:43.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:43 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:43.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:43 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:45.535 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:45 smithi119 ceph-mon[21835]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:45.535 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:45 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:07:45.535 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:45 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:45.535 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:45 smithi119 ceph-mon[21835]: Deploying daemon osd.2 on smithi119 2024-04-22T15:07:45.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:45 smithi186 ceph-mon[24041]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:45.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:45 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:07:45.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:45 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:45.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:45 smithi186 ceph-mon[24041]: Deploying daemon osd.2 on smithi119 2024-04-22T15:07:47.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:47 smithi119 ceph-mon[21835]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:47.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:47 smithi186 ceph-mon[24041]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:49.614 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:49 smithi119 ceph-mon[21835]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:49.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:49 smithi186 ceph-mon[24041]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:50.978 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:50.979 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:50.979 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:50.979 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:50.979 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:07:50.979 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:07:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:07:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:07:51.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:51 smithi119 ceph-mon[21835]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:51 smithi186 ceph-mon[24041]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:53.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:53 smithi119 ceph-mon[21835]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:53.660 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:53 smithi119 ceph-mon[21835]: from='osd.2 [v2:172.21.15.119:6818/1451810811,v1:172.21.15.119:6819/1451810811]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:07:53.691 INFO:teuthology.orchestra.run.smithi119.stdout:Created osd(s) 2 on host 'smithi119' 2024-04-22T15:07:53.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:53 smithi186 ceph-mon[24041]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:53.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:53 smithi186 ceph-mon[24041]: from='osd.2 [v2:172.21.15.119:6818/1451810811,v1:172.21.15.119:6819/1451810811]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:07:54.287 DEBUG:teuthology.orchestra.run.smithi119:osd.2> sudo journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.2.service 2024-04-22T15:07:54.289 INFO:tasks.cephadm:Deploying osd.3 on smithi119 with /dev/vg_nvme/lv_1... 2024-04-22T15:07:54.290 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-22T15:07:54.825 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:54 smithi119 ceph-mon[21835]: from='osd.2 [v2:172.21.15.119:6818/1451810811,v1:172.21.15.119:6819/1451810811]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T15:07:54.825 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:54 smithi119 ceph-mon[21835]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T15:07:54.825 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:54 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:54.825 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:54 smithi119 ceph-mon[21835]: from='osd.2 [v2:172.21.15.119:6818/1451810811,v1:172.21.15.119:6819/1451810811]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:07:54.825 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:54 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:54.826 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:07:54 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2[40001]: 2024-04-22T15:07:54.602+0000 7f8dc756e700 -1 osd.2 0 waiting for initial osdmap 2024-04-22T15:07:54.826 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:07:54 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2[40001]: 2024-04-22T15:07:54.607+0000 7f8dc1f05700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:07:55.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:54 smithi186 ceph-mon[24041]: from='osd.2 [v2:172.21.15.119:6818/1451810811,v1:172.21.15.119:6819/1451810811]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T15:07:55.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:54 smithi186 ceph-mon[24041]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T15:07:55.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:54 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:55.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:54 smithi186 ceph-mon[24041]: from='osd.2 [v2:172.21.15.119:6818/1451810811,v1:172.21.15.119:6819/1451810811]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:07:55.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:54 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:55.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:55 smithi119 ceph-mon[21835]: pgmap v67: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:55.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:55 smithi119 ceph-mon[21835]: from='osd.2 [v2:172.21.15.119:6818/1451810811,v1:172.21.15.119:6819/1451810811]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]': finished 2024-04-22T15:07:55.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:55 smithi119 ceph-mon[21835]: osdmap e18: 3 total, 2 up, 3 in 2024-04-22T15:07:55.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:55 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:55.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:55 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:55 smithi186 ceph-mon[24041]: pgmap v67: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:56.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:55 smithi186 ceph-mon[24041]: from='osd.2 [v2:172.21.15.119:6818/1451810811,v1:172.21.15.119:6819/1451810811]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]': finished 2024-04-22T15:07:56.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:55 smithi186 ceph-mon[24041]: osdmap e18: 3 total, 2 up, 3 in 2024-04-22T15:07:56.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:55 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:56.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:55 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:56.915 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:56 smithi119 ceph-mon[21835]: purged_snaps scrub starts 2024-04-22T15:07:56.915 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:56 smithi119 ceph-mon[21835]: purged_snaps scrub ok 2024-04-22T15:07:56.916 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:56 smithi119 ceph-mon[21835]: osd.2 [v2:172.21.15.119:6818/1451810811,v1:172.21.15.119:6819/1451810811] boot 2024-04-22T15:07:56.916 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:56 smithi119 ceph-mon[21835]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T15:07:56.916 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:56.916 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T15:07:57.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:56 smithi186 ceph-mon[24041]: purged_snaps scrub starts 2024-04-22T15:07:57.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:56 smithi186 ceph-mon[24041]: purged_snaps scrub ok 2024-04-22T15:07:57.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:56 smithi186 ceph-mon[24041]: osd.2 [v2:172.21.15.119:6818/1451810811,v1:172.21.15.119:6819/1451810811] boot 2024-04-22T15:07:57.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:56 smithi186 ceph-mon[24041]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T15:07:57.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:57.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T15:07:57.078 INFO:teuthology.orchestra.run.smithi119.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-22T15:07:57.078 INFO:teuthology.orchestra.run.smithi119.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-22T15:07:57.078 INFO:teuthology.orchestra.run.smithi119.stderr: stderr: 10+0 records in 2024-04-22T15:07:57.078 INFO:teuthology.orchestra.run.smithi119.stderr:10+0 records out 2024-04-22T15:07:57.078 INFO:teuthology.orchestra.run.smithi119.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0123771 s, 847 MB/s 2024-04-22T15:07:57.078 INFO:teuthology.orchestra.run.smithi119.stderr:--> Zapping successful for: 2024-04-22T15:07:57.721 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:57 smithi119 ceph-mon[21835]: pgmap v70: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:57.722 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:57 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T15:07:57.722 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:57 smithi119 ceph-mon[21835]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T15:07:57.722 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:57 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T15:07:57.746 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi119:vg_nvme/lv_1 2024-04-22T15:07:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:57 smithi186 ceph-mon[24041]: pgmap v70: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:57 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T15:07:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:57 smithi186 ceph-mon[24041]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T15:07:58.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:57 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T15:07:58.371 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:07:58 smithi119 sudo[41335]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T15:07:58.371 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:07:58 smithi119 sudo[41335]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:58.371 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:07:58 smithi119 sudo[41335]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:58.635 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:07:58 smithi119 sudo[41345]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T15:07:58.636 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:07:58 smithi119 sudo[41345]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:58.636 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:07:58 smithi119 sudo[41345]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:58.905 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:07:58 smithi119 sudo[41352]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T15:07:58.905 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:07:58 smithi119 sudo[41352]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:58.905 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:07:58 smithi119 sudo[41352]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:58.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:58 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T15:07:58.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:58 smithi119 ceph-mon[21835]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T15:07:58.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:58 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:59.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:58 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T15:07:59.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:58 smithi186 ceph-mon[24041]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T15:07:59.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:58 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:07:59.194 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:07:58 smithi119 sudo[41400]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T15:07:59.194 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:07:58 smithi119 sudo[41400]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:59.194 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:07:58 smithi119 sudo[41400]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:59.444 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:07:59 smithi119 sudo[41407]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T15:07:59.444 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:07:59 smithi119 sudo[41407]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:59.444 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:07:59 smithi119 sudo[41407]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:59.702 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:59 smithi119 sudo[41445]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T15:07:59.702 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:59 smithi119 sudo[41445]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:59.702 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:07:59 smithi119 sudo[41424]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T15:07:59.702 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:07:59 smithi119 sudo[41424]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:59.702 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:07:59 smithi119 sudo[41424]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:59.955 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:59 smithi119 sudo[41445]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:59.955 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:59 smithi119 sudo[41455]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T15:07:59.955 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:59 smithi119 sudo[41455]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:59.955 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:59 smithi119 sudo[41455]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:59.955 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:59 smithi119 ceph-mon[21835]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:59.956 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:07:59 smithi119 ceph-mon[21835]: osdmap e22: 3 total, 3 up, 3 in 2024-04-22T15:08:00.210 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:59 smithi186 ceph-mon[24041]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:00.210 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:07:59 smithi186 ceph-mon[24041]: osdmap e22: 3 total, 3 up, 3 in 2024-04-22T15:08:00.210 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 sudo[26267]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T15:08:00.210 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 sudo[26267]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:08:00.210 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 sudo[26267]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:08:00.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 sudo[26270]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T15:08:00.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 sudo[26270]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:08:00.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 sudo[26270]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:08:01.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:08:01.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:08:01.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:08:01.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:08:01.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:08:01.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:08:01.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:08:01.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:01.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi119:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:08:01.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:08:01.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:08:01.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:01.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:08:01.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: mgrmap e19: smithi119.jjbxxd(active, since 2m), standbys: smithi186.vtjgwg 2024-04-22T15:08:01.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:01.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:00 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:01.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:08:01.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi119:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: mgrmap e19: smithi119.jjbxxd(active, since 2m), standbys: smithi186.vtjgwg 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:01.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:00 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:03.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:03 smithi119 ceph-mon[21835]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:03.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:03 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2850606205' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e3c166b6-455e-4900-ae71-472de8a7fdb5"}]: dispatch 2024-04-22T15:08:03.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:03 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2850606205' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e3c166b6-455e-4900-ae71-472de8a7fdb5"}]': finished 2024-04-22T15:08:03.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:03 smithi119 ceph-mon[21835]: osdmap e23: 4 total, 3 up, 4 in 2024-04-22T15:08:03.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:03 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:08:03.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:03 smithi186 ceph-mon[24041]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:03.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:03 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2850606205' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e3c166b6-455e-4900-ae71-472de8a7fdb5"}]: dispatch 2024-04-22T15:08:03.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:03 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2850606205' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e3c166b6-455e-4900-ae71-472de8a7fdb5"}]': finished 2024-04-22T15:08:03.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:03 smithi186 ceph-mon[24041]: osdmap e23: 4 total, 3 up, 4 in 2024-04-22T15:08:03.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:03 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:08:04.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:04 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/4260137261' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:04.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:04 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/4260137261' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:05.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:05 smithi186 ceph-mon[24041]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:05.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:05 smithi119 ceph-mon[21835]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:07.712 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:07 smithi119 ceph-mon[21835]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:07.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:07 smithi186 ceph-mon[24041]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:09.604 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:09 smithi186 ceph-mon[24041]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:09.605 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:09 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:09.605 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:09 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:09.605 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:09 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:09.605 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:09 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:09.605 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:09 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:09.605 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:09 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:09.605 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:09 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:09.605 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:09 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:09.605 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:09 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:09.767 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:09 smithi119 ceph-mon[21835]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:09.767 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:09 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:09.767 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:09 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:09.767 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:09 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:09.767 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:09 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:09.767 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:09 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:09.767 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:09 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:09.767 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:09 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:09.767 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:09 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:09.768 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:09 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:10.738 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:10 smithi119 ceph-mon[21835]: Detected new or changed devices on smithi119 2024-04-22T15:08:10.738 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:10 smithi119 ceph-mon[21835]: Adjusting osd_memory_target on smithi119 to 4288M 2024-04-22T15:08:10.738 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:10 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:10 smithi186 ceph-mon[24041]: Detected new or changed devices on smithi119 2024-04-22T15:08:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:10 smithi186 ceph-mon[24041]: Adjusting osd_memory_target on smithi119 to 4288M 2024-04-22T15:08:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:10 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:11.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:11 smithi119 ceph-mon[21835]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:11.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:08:11.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:11 smithi186 ceph-mon[24041]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:08:12.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:12.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:12 smithi119 ceph-mon[21835]: Deploying daemon osd.3 on smithi119 2024-04-22T15:08:13.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:12 smithi186 ceph-mon[24041]: Deploying daemon osd.3 on smithi119 2024-04-22T15:08:13.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:13 smithi186 ceph-mon[24041]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:13.937 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:13 smithi119 ceph-mon[21835]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:15.822 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:15 smithi119 ceph-mon[21835]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:16.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:15 smithi186 ceph-mon[24041]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:17.212 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:16 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:17.212 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:16 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:17.212 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:16 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:17.212 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:16 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:17.212 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:16 smithi119 ceph-mon[21835]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:16 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:16 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:16 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:16 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:17.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:16 smithi186 ceph-mon[24041]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:19.048 INFO:teuthology.orchestra.run.smithi119.stdout:Created osd(s) 3 on host 'smithi119' 2024-04-22T15:08:19.396 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:19 smithi119 ceph-mon[21835]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:19.396 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:19 smithi119 ceph-mon[21835]: from='osd.3 [v2:172.21.15.119:6826/602651652,v1:172.21.15.119:6827/602651652]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:08:19.396 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:19 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:19.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:19 smithi186 ceph-mon[24041]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:19.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:19 smithi186 ceph-mon[24041]: from='osd.3 [v2:172.21.15.119:6826/602651652,v1:172.21.15.119:6827/602651652]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:08:19.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:19 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:19.694 DEBUG:teuthology.orchestra.run.smithi119:osd.3> sudo journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.3.service 2024-04-22T15:08:19.696 INFO:tasks.cephadm:Deploying osd.4 on smithi186 with /dev/vg_nvme/lv_4... 2024-04-22T15:08:19.696 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-22T15:08:20.200 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:20 smithi119 ceph-mon[21835]: from='osd.3 [v2:172.21.15.119:6826/602651652,v1:172.21.15.119:6827/602651652]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T15:08:20.200 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:20 smithi119 ceph-mon[21835]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T15:08:20.200 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:20 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:08:20.200 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:20 smithi119 ceph-mon[21835]: from='osd.3 [v2:172.21.15.119:6826/602651652,v1:172.21.15.119:6827/602651652]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:08:20.200 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:08:20 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[43264]: 2024-04-22T15:08:20.159+0000 7f4a5d094700 -1 osd.3 0 waiting for initial osdmap 2024-04-22T15:08:20.201 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:08:20 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[43264]: 2024-04-22T15:08:20.164+0000 7f4a5722a700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:08:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:20 smithi186 ceph-mon[24041]: from='osd.3 [v2:172.21.15.119:6826/602651652,v1:172.21.15.119:6827/602651652]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T15:08:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:20 smithi186 ceph-mon[24041]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T15:08:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:20 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:08:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:20 smithi186 ceph-mon[24041]: from='osd.3 [v2:172.21.15.119:6826/602651652,v1:172.21.15.119:6827/602651652]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:08:21.422 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:21 smithi119 ceph-mon[21835]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:21.422 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:21 smithi119 ceph-mon[21835]: from='osd.3 [v2:172.21.15.119:6826/602651652,v1:172.21.15.119:6827/602651652]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]': finished 2024-04-22T15:08:21.422 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:21 smithi119 ceph-mon[21835]: osdmap e25: 4 total, 3 up, 4 in 2024-04-22T15:08:21.422 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:21 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:08:21.423 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:21 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:08:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:21 smithi186 ceph-mon[24041]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:21 smithi186 ceph-mon[24041]: from='osd.3 [v2:172.21.15.119:6826/602651652,v1:172.21.15.119:6827/602651652]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]': finished 2024-04-22T15:08:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:21 smithi186 ceph-mon[24041]: osdmap e25: 4 total, 3 up, 4 in 2024-04-22T15:08:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:21 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:08:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:21 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:08:21.638 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-22T15:08:21.639 INFO:teuthology.orchestra.run.smithi186.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-22T15:08:21.639 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10+0 records in 2024-04-22T15:08:21.639 INFO:teuthology.orchestra.run.smithi186.stderr:10+0 records out 2024-04-22T15:08:21.639 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0137848 s, 761 MB/s 2024-04-22T15:08:21.639 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping successful for: 2024-04-22T15:08:22.251 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi186:vg_nvme/lv_4 2024-04-22T15:08:22.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:22 smithi119 ceph-mon[21835]: purged_snaps scrub starts 2024-04-22T15:08:22.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:22 smithi119 ceph-mon[21835]: purged_snaps scrub ok 2024-04-22T15:08:22.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:22 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:08:22.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:22 smithi119 ceph-mon[21835]: osd.3 [v2:172.21.15.119:6826/602651652,v1:172.21.15.119:6827/602651652] boot 2024-04-22T15:08:22.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:22 smithi119 ceph-mon[21835]: osdmap e26: 4 total, 4 up, 4 in 2024-04-22T15:08:22.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:22 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:08:22.505 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:22 smithi186 ceph-mon[24041]: purged_snaps scrub starts 2024-04-22T15:08:22.505 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:22 smithi186 ceph-mon[24041]: purged_snaps scrub ok 2024-04-22T15:08:22.505 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:22 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:08:22.505 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:22 smithi186 ceph-mon[24041]: osd.3 [v2:172.21.15.119:6826/602651652,v1:172.21.15.119:6827/602651652] boot 2024-04-22T15:08:22.505 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:22 smithi186 ceph-mon[24041]: osdmap e26: 4 total, 4 up, 4 in 2024-04-22T15:08:22.506 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:22 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:08:23.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:23 smithi119 ceph-mon[21835]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:23.762 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:23 smithi119 ceph-mon[21835]: osdmap e27: 4 total, 4 up, 4 in 2024-04-22T15:08:23.762 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:23 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:23.763 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:23 smithi186 ceph-mon[24041]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:08:23.763 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:23 smithi186 ceph-mon[24041]: osdmap e27: 4 total, 4 up, 4 in 2024-04-22T15:08:23.763 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:23 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:24.705 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:24 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:08:24.705 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:24 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:08:24.705 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:24 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:24.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:24 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:08:24.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:24 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:08:24.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:24 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:25.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:25 smithi119 ceph-mon[21835]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:25.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:25 smithi119 ceph-mon[21835]: from='client.24137 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:08:25.767 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:25 smithi186 ceph-mon[24041]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:25.768 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:25 smithi186 ceph-mon[24041]: from='client.24137 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:08:27.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:27.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:27.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:27.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:27.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:27.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:27.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: Adjusting osd_memory_target on smithi119 to 3216M 2024-04-22T15:08:27.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:27.493 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:27.493 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:27.493 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:27.493 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:27.493 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:27.493 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/2760677389' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7de45c9-0187-4e07-8980-90e599176928"}]: dispatch 2024-04-22T15:08:27.493 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7de45c9-0187-4e07-8980-90e599176928"}]: dispatch 2024-04-22T15:08:27.493 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a7de45c9-0187-4e07-8980-90e599176928"}]': finished 2024-04-22T15:08:27.493 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: osdmap e28: 5 total, 4 up, 5 in 2024-04-22T15:08:27.493 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:27.493 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:27 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/1133469312' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:27.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: Adjusting osd_memory_target on smithi119 to 3216M 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:27.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/2760677389' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7de45c9-0187-4e07-8980-90e599176928"}]: dispatch 2024-04-22T15:08:27.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7de45c9-0187-4e07-8980-90e599176928"}]: dispatch 2024-04-22T15:08:27.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a7de45c9-0187-4e07-8980-90e599176928"}]': finished 2024-04-22T15:08:27.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: osdmap e28: 5 total, 4 up, 5 in 2024-04-22T15:08:27.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:27.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:27 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/1133469312' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:29.440 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:29 smithi186 ceph-mon[24041]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:29.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:29 smithi119 ceph-mon[21835]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:31.463 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:31 smithi186 ceph-mon[24041]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:31.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:31 smithi119 ceph-mon[21835]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:32.711 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:32 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:08:32.711 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:32 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:32.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:32 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:08:32.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:32 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:33.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:33 smithi186 ceph-mon[24041]: Deploying daemon osd.4 on smithi186 2024-04-22T15:08:33.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:33 smithi186 ceph-mon[24041]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:33.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:33 smithi119 ceph-mon[21835]: Deploying daemon osd.4 on smithi186 2024-04-22T15:08:33.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:33 smithi119 ceph-mon[21835]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:35.515 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:35 smithi186 ceph-mon[24041]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:35.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:35 smithi119 ceph-mon[21835]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:37.620 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:37 smithi186 ceph-mon[24041]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:37.620 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:37.620 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:37.620 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:37.620 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:37.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:37 smithi119 ceph-mon[21835]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:37.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:37.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:37.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:37.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:39.683 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:39 smithi186 ceph-mon[24041]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:39.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:39 smithi119 ceph-mon[21835]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:40.366 INFO:teuthology.orchestra.run.smithi186.stdout:Created osd(s) 4 on host 'smithi186' 2024-04-22T15:08:40.608 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:40 smithi186 ceph-mon[24041]: from='osd.4 [v2:172.21.15.186:6800/3140332101,v1:172.21.15.186:6801/3140332101]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:08:40.608 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:40 smithi186 ceph-mon[24041]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:08:40.608 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:40 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:40.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:40 smithi119 ceph-mon[21835]: from='osd.4 [v2:172.21.15.186:6800/3140332101,v1:172.21.15.186:6801/3140332101]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:08:40.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:40 smithi119 ceph-mon[21835]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:08:40.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:40 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:41.250 DEBUG:teuthology.orchestra.run.smithi186:osd.4> sudo journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.4.service 2024-04-22T15:08:41.252 INFO:tasks.cephadm:Deploying osd.5 on smithi186 with /dev/vg_nvme/lv_3... 2024-04-22T15:08:41.252 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-22T15:08:41.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:41 smithi186 ceph-mon[24041]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:41.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:41 smithi186 ceph-mon[24041]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T15:08:41.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:41 smithi186 ceph-mon[24041]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T15:08:41.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:41 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:41.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:41 smithi186 ceph-mon[24041]: from='osd.4 [v2:172.21.15.186:6800/3140332101,v1:172.21.15.186:6801/3140332101]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:08:41.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:41 smithi186 ceph-mon[24041]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:08:41.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:41 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:41.776 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:08:41 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[28181]: 2024-04-22T15:08:41.518+0000 7fe7f5f1f700 -1 osd.4 0 waiting for initial osdmap 2024-04-22T15:08:41.776 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:08:41 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[28181]: 2024-04-22T15:08:41.524+0000 7fe7f08b6700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:08:41.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:41 smithi119 ceph-mon[21835]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:41.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:41 smithi119 ceph-mon[21835]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T15:08:41.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:41 smithi119 ceph-mon[21835]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T15:08:41.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:41 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:41.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:41 smithi119 ceph-mon[21835]: from='osd.4 [v2:172.21.15.186:6800/3140332101,v1:172.21.15.186:6801/3140332101]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:08:41.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:41 smithi119 ceph-mon[21835]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:08:41.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:41 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:42 smithi186 ceph-mon[24041]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-04-22T15:08:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:42 smithi186 ceph-mon[24041]: osdmap e30: 5 total, 4 up, 5 in 2024-04-22T15:08:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:42 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:42.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:42 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:42.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:42 smithi186 ceph-mon[24041]: osd.4 [v2:172.21.15.186:6800/3140332101,v1:172.21.15.186:6801/3140332101] boot 2024-04-22T15:08:42.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:42 smithi186 ceph-mon[24041]: osdmap e31: 5 total, 5 up, 5 in 2024-04-22T15:08:42.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:42 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:42.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:42 smithi119 ceph-mon[21835]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-04-22T15:08:42.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:42 smithi119 ceph-mon[21835]: osdmap e30: 5 total, 4 up, 5 in 2024-04-22T15:08:42.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:42 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:42.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:42 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:42.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:42 smithi119 ceph-mon[21835]: osd.4 [v2:172.21.15.186:6800/3140332101,v1:172.21.15.186:6801/3140332101] boot 2024-04-22T15:08:42.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:42 smithi119 ceph-mon[21835]: osdmap e31: 5 total, 5 up, 5 in 2024-04-22T15:08:42.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:42 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:43.835 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:43 smithi186 ceph-mon[24041]: purged_snaps scrub starts 2024-04-22T15:08:43.836 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:43 smithi186 ceph-mon[24041]: purged_snaps scrub ok 2024-04-22T15:08:43.836 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:43 smithi186 ceph-mon[24041]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:43.836 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:43 smithi186 ceph-mon[24041]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T15:08:43.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:43 smithi119 ceph-mon[21835]: purged_snaps scrub starts 2024-04-22T15:08:43.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:43 smithi119 ceph-mon[21835]: purged_snaps scrub ok 2024-04-22T15:08:43.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:43 smithi119 ceph-mon[21835]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:43.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:43 smithi119 ceph-mon[21835]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T15:08:44.931 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-22T15:08:44.931 INFO:teuthology.orchestra.run.smithi186.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-22T15:08:44.932 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10+0 records in 2024-04-22T15:08:44.932 INFO:teuthology.orchestra.run.smithi186.stderr:10+0 records out 2024-04-22T15:08:44.932 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0139515 s, 752 MB/s 2024-04-22T15:08:44.932 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping successful for: 2024-04-22T15:08:45.638 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:45 smithi186 ceph-mon[24041]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:45.638 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:45 smithi186 ceph-mon[24041]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T15:08:45.639 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:45 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:45.639 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:45 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:45.639 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:45 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:45.639 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:45 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:45.639 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:45 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:45.639 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:45 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:45.639 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:45 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:45.639 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:45 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:45.660 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi186:vg_nvme/lv_3 2024-04-22T15:08:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:45 smithi119 ceph-mon[21835]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:45 smithi119 ceph-mon[21835]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T15:08:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:45 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:45 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:45 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:45 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:45 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:45 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:45 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:45 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:08:46.856 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:46 smithi186 ceph-mon[24041]: Detected new or changed devices on smithi186 2024-04-22T15:08:46.856 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:46 smithi186 ceph-mon[24041]: Adjusting osd_memory_target on smithi186 to 15936M 2024-04-22T15:08:46.856 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:46 smithi186 ceph-mon[24041]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T15:08:46.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:46 smithi119 ceph-mon[21835]: Detected new or changed devices on smithi186 2024-04-22T15:08:46.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:46 smithi119 ceph-mon[21835]: Adjusting osd_memory_target on smithi186 to 15936M 2024-04-22T15:08:46.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:46 smithi119 ceph-mon[21835]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T15:08:47.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:47 smithi186 ceph-mon[24041]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-22T15:08:47.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:47 smithi186 ceph-mon[24041]: osdmap e35: 5 total, 5 up, 5 in 2024-04-22T15:08:47.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:47 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:08:47.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:47 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:08:47.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:47 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:47.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:47 smithi119 ceph-mon[21835]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-22T15:08:47.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:47 smithi119 ceph-mon[21835]: osdmap e35: 5 total, 5 up, 5 in 2024-04-22T15:08:47.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:47 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:08:47.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:47 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:08:47.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:47 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:48.832 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:48 smithi186 ceph-mon[24041]: from='client.24155 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:08:48.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:48 smithi119 ceph-mon[21835]: from='client.24155 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:08:49.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:49 smithi119 ceph-mon[21835]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-22T15:08:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:49 smithi186 ceph-mon[24041]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-22T15:08:50.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:50 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/2866558087' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea7cee92-1fa1-4f5b-9b42-ca5673853759"}]: dispatch 2024-04-22T15:08:50.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:50 smithi119 ceph-mon[21835]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea7cee92-1fa1-4f5b-9b42-ca5673853759"}]: dispatch 2024-04-22T15:08:50.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:50 smithi119 ceph-mon[21835]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea7cee92-1fa1-4f5b-9b42-ca5673853759"}]': finished 2024-04-22T15:08:50.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:50 smithi119 ceph-mon[21835]: osdmap e36: 6 total, 5 up, 6 in 2024-04-22T15:08:50.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:50.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:08:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:08:50.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:50 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/1190718601' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:50 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/2866558087' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea7cee92-1fa1-4f5b-9b42-ca5673853759"}]: dispatch 2024-04-22T15:08:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:50 smithi186 ceph-mon[24041]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea7cee92-1fa1-4f5b-9b42-ca5673853759"}]: dispatch 2024-04-22T15:08:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:50 smithi186 ceph-mon[24041]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea7cee92-1fa1-4f5b-9b42-ca5673853759"}]': finished 2024-04-22T15:08:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:50 smithi186 ceph-mon[24041]: osdmap e36: 6 total, 5 up, 6 in 2024-04-22T15:08:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:08:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:08:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:50 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/1190718601' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:51.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:51 smithi119 ceph-mon[21835]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-04-22T15:08:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:51 smithi186 ceph-mon[24041]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-04-22T15:08:53.852 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:53 smithi186 ceph-mon[24041]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-04-22T15:08:53.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:53 smithi119 ceph-mon[21835]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-04-22T15:08:55.855 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:55 smithi186 ceph-mon[24041]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:55.948 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:55 smithi119 ceph-mon[21835]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:56.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:08:56.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:57.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:08:57.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:57.762 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:57 smithi186 ceph-mon[24041]: Deploying daemon osd.5 on smithi186 2024-04-22T15:08:57.763 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:57 smithi186 ceph-mon[24041]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:57.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:57 smithi119 ceph-mon[21835]: Deploying daemon osd.5 on smithi186 2024-04-22T15:08:57.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:57 smithi119 ceph-mon[21835]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:59.933 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:08:59 smithi186 ceph-mon[24041]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:59.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:08:59 smithi119 ceph-mon[21835]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:09:01.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:01 smithi186 ceph-mon[24041]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:09:01.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:01 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:01.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:01 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:01.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:01 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:01.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:01 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:01.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:01 smithi119 ceph-mon[21835]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:09:01.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:01 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:01.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:01 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:01.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:01 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:01.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:01 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:03.834 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:03 smithi186 ceph-mon[24041]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:09:03.843 INFO:teuthology.orchestra.run.smithi186.stdout:Created osd(s) 5 on host 'smithi186' 2024-04-22T15:09:03.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:03 smithi119 ceph-mon[21835]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:09:04.505 DEBUG:teuthology.orchestra.run.smithi186:osd.5> sudo journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.5.service 2024-04-22T15:09:04.507 INFO:tasks.cephadm:Deploying osd.6 on smithi186 with /dev/vg_nvme/lv_2... 2024-04-22T15:09:04.507 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-22T15:09:05.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:04 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:05.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:04 smithi186 ceph-mon[24041]: from='osd.5 [v2:172.21.15.186:6808/3541622350,v1:172.21.15.186:6809/3541622350]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:09:05.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:04 smithi186 ceph-mon[24041]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:09:05.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:04 smithi186 ceph-mon[24041]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:09:05.110 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:04 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:05.111 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:04 smithi119 ceph-mon[21835]: from='osd.5 [v2:172.21.15.186:6808/3541622350,v1:172.21.15.186:6809/3541622350]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:09:05.111 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:04 smithi119 ceph-mon[21835]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:09:05.111 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:04 smithi119 ceph-mon[21835]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:09:06.184 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:05 smithi186 ceph-mon[24041]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T15:09:06.184 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:05 smithi186 ceph-mon[24041]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T15:09:06.184 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:05 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:09:06.184 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:05 smithi186 ceph-mon[24041]: from='osd.5 [v2:172.21.15.186:6808/3541622350,v1:172.21.15.186:6809/3541622350]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:09:06.184 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:05 smithi186 ceph-mon[24041]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:09:06.185 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:09:05 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[31169]: 2024-04-22T15:09:05.853+0000 7fb96ac99700 -1 osd.5 0 waiting for initial osdmap 2024-04-22T15:09:06.185 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:09:05 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[31169]: 2024-04-22T15:09:05.858+0000 7fb96462e700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:09:06.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:05 smithi119 ceph-mon[21835]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T15:09:06.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:05 smithi119 ceph-mon[21835]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T15:09:06.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:05 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:09:06.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:05 smithi119 ceph-mon[21835]: from='osd.5 [v2:172.21.15.186:6808/3541622350,v1:172.21.15.186:6809/3541622350]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:09:06.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:05 smithi119 ceph-mon[21835]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:09:06.953 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-22T15:09:06.953 INFO:teuthology.orchestra.run.smithi186.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-22T15:09:06.953 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10+0 records in 2024-04-22T15:09:06.953 INFO:teuthology.orchestra.run.smithi186.stderr:10+0 records out 2024-04-22T15:09:06.953 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125959 s, 832 MB/s 2024-04-22T15:09:06.953 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping successful for: 2024-04-22T15:09:07.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:06 smithi119 ceph-mon[21835]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-04-22T15:09:07.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:06 smithi119 ceph-mon[21835]: osdmap e38: 6 total, 5 up, 6 in 2024-04-22T15:09:07.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:06 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:09:07.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:06 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:09:07.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:06 smithi119 ceph-mon[21835]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:09:07.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:06 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:07.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:06 smithi119 ceph-mon[21835]: osd.5 [v2:172.21.15.186:6808/3541622350,v1:172.21.15.186:6809/3541622350] boot 2024-04-22T15:09:07.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:06 smithi119 ceph-mon[21835]: osdmap e39: 6 total, 6 up, 6 in 2024-04-22T15:09:07.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:06 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:09:07.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:06 smithi186 ceph-mon[24041]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-04-22T15:09:07.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:06 smithi186 ceph-mon[24041]: osdmap e38: 6 total, 5 up, 6 in 2024-04-22T15:09:07.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:09:07.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:09:07.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:06 smithi186 ceph-mon[24041]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:09:07.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:07.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:06 smithi186 ceph-mon[24041]: osd.5 [v2:172.21.15.186:6808/3541622350,v1:172.21.15.186:6809/3541622350] boot 2024-04-22T15:09:07.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:06 smithi186 ceph-mon[24041]: osdmap e39: 6 total, 6 up, 6 in 2024-04-22T15:09:07.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:06 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:09:07.960 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:07 smithi186 ceph-mon[24041]: purged_snaps scrub starts 2024-04-22T15:09:07.960 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:07 smithi186 ceph-mon[24041]: purged_snaps scrub ok 2024-04-22T15:09:07.960 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:07 smithi186 ceph-mon[24041]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T15:09:07.961 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi186:vg_nvme/lv_2 2024-04-22T15:09:08.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:07 smithi119 ceph-mon[21835]: purged_snaps scrub starts 2024-04-22T15:09:08.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:07 smithi119 ceph-mon[21835]: purged_snaps scrub ok 2024-04-22T15:09:08.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:07 smithi119 ceph-mon[21835]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T15:09:09.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:09 smithi186 ceph-mon[24041]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:09.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:09 smithi186 ceph-mon[24041]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T15:09:09.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:09 smithi119 ceph-mon[21835]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:09.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:09 smithi119 ceph-mon[21835]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T15:09:10.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:10 smithi186 ceph-mon[24041]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T15:09:10.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:10 smithi119 ceph-mon[21835]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T15:09:11.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: pgmap v130: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:11.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: osdmap e43: 6 total, 6 up, 6 in 2024-04-22T15:09:11.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:09:11.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:09:11.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:11.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:11.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:11.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:11.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:11.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:11.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:11.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:11.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:11.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: pgmap v130: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: osdmap e43: 6 total, 6 up, 6 in 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:11.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:12.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:12 smithi186 ceph-mon[24041]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:12.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:12 smithi186 ceph-mon[24041]: Detected new or changed devices on smithi186 2024-04-22T15:09:12.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:12 smithi186 ceph-mon[24041]: Adjusting osd_memory_target on smithi186 to 7968M 2024-04-22T15:09:12.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:12 smithi119 ceph-mon[21835]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:12.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:12 smithi119 ceph-mon[21835]: Detected new or changed devices on smithi186 2024-04-22T15:09:12.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:12 smithi119 ceph-mon[21835]: Adjusting osd_memory_target on smithi186 to 7968M 2024-04-22T15:09:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:13 smithi186 ceph-mon[24041]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:13 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/48444007' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ea83fed-ee69-4db5-9ed9-9912da48ff93"}]: dispatch 2024-04-22T15:09:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:13 smithi186 ceph-mon[24041]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ea83fed-ee69-4db5-9ed9-9912da48ff93"}]: dispatch 2024-04-22T15:09:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:13 smithi186 ceph-mon[24041]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ea83fed-ee69-4db5-9ed9-9912da48ff93"}]': finished 2024-04-22T15:09:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:13 smithi186 ceph-mon[24041]: osdmap e44: 7 total, 6 up, 7 in 2024-04-22T15:09:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:13 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:13.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:13 smithi119 ceph-mon[21835]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:13.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:13 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/48444007' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ea83fed-ee69-4db5-9ed9-9912da48ff93"}]: dispatch 2024-04-22T15:09:13.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:13 smithi119 ceph-mon[21835]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ea83fed-ee69-4db5-9ed9-9912da48ff93"}]: dispatch 2024-04-22T15:09:13.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:13 smithi119 ceph-mon[21835]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ea83fed-ee69-4db5-9ed9-9912da48ff93"}]': finished 2024-04-22T15:09:13.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:13 smithi119 ceph-mon[21835]: osdmap e44: 7 total, 6 up, 7 in 2024-04-22T15:09:13.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:13 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:14.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:14 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/3785239947' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:09:14.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:14 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/3785239947' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:09:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:15 smithi186 ceph-mon[24041]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:15.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:15 smithi119 ceph-mon[21835]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:17.555 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:17 smithi186 ceph-mon[24041]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:17.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:17 smithi119 ceph-mon[21835]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:19.874 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:19 smithi186 ceph-mon[24041]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:19.874 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:19 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:09:19.874 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:19 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:19.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:19 smithi119 ceph-mon[21835]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:19.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:19 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:09:19.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:19 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:20.851 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:20 smithi186 ceph-mon[24041]: Deploying daemon osd.6 on smithi186 2024-04-22T15:09:20.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:20 smithi119 ceph-mon[21835]: Deploying daemon osd.6 on smithi186 2024-04-22T15:09:21.947 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:21 smithi186 ceph-mon[24041]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:21.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:21 smithi119 ceph-mon[21835]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:23.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:23 smithi186 ceph-mon[24041]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:23.793 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:23 smithi119 ceph-mon[21835]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:25.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:25 smithi186 ceph-mon[24041]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:25.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:25 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:25 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:25 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:25 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:25.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:25 smithi119 ceph-mon[21835]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:25.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:25 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:25.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:25 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:25.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:25 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:25.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:25 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:27.355 INFO:teuthology.orchestra.run.smithi186.stdout:Created osd(s) 6 on host 'smithi186' 2024-04-22T15:09:27.676 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:27 smithi186 ceph-mon[24041]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:27.676 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:27 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:27.676 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:27 smithi186 ceph-mon[24041]: from='osd.6 [v2:172.21.15.186:6816/2393038,v1:172.21.15.186:6817/2393038]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:09:27.676 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:27 smithi186 ceph-mon[24041]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:09:27.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:27 smithi119 ceph-mon[21835]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:27.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:27 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:27.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:27 smithi119 ceph-mon[21835]: from='osd.6 [v2:172.21.15.186:6816/2393038,v1:172.21.15.186:6817/2393038]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:09:27.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:27 smithi119 ceph-mon[21835]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:09:28.078 DEBUG:teuthology.orchestra.run.smithi186:osd.6> sudo journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.6.service 2024-04-22T15:09:28.081 INFO:tasks.cephadm:Deploying osd.7 on smithi186 with /dev/vg_nvme/lv_1... 2024-04-22T15:09:28.081 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-22T15:09:29.526 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:09:29 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[34211]: 2024-04-22T15:09:29.364+0000 7f3bc7e9d700 -1 osd.6 0 waiting for initial osdmap 2024-04-22T15:09:29.526 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:09:29 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[34211]: 2024-04-22T15:09:29.370+0000 7f3bc2033700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:09:29.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:29 smithi186 ceph-mon[24041]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:29.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:29 smithi186 ceph-mon[24041]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T15:09:29.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:29 smithi186 ceph-mon[24041]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T15:09:29.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:29 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:29.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:29 smithi186 ceph-mon[24041]: from='osd.6 [v2:172.21.15.186:6816/2393038,v1:172.21.15.186:6817/2393038]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:09:29.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:29 smithi186 ceph-mon[24041]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:09:29.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:29 smithi119 ceph-mon[21835]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:29.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:29 smithi119 ceph-mon[21835]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T15:09:29.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:29 smithi119 ceph-mon[21835]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T15:09:29.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:29 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:29.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:29 smithi119 ceph-mon[21835]: from='osd.6 [v2:172.21.15.186:6816/2393038,v1:172.21.15.186:6817/2393038]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:09:29.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:29 smithi119 ceph-mon[21835]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:09:30.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:30 smithi186 ceph-mon[24041]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-04-22T15:09:30.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:30 smithi186 ceph-mon[24041]: osdmap e46: 7 total, 6 up, 7 in 2024-04-22T15:09:30.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:30 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:30.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:30 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:30.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:30 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:30.629 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-22T15:09:30.630 INFO:teuthology.orchestra.run.smithi186.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-22T15:09:30.630 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10+0 records in 2024-04-22T15:09:30.630 INFO:teuthology.orchestra.run.smithi186.stderr:10+0 records out 2024-04-22T15:09:30.630 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0126832 s, 827 MB/s 2024-04-22T15:09:30.630 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping successful for: 2024-04-22T15:09:30.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:30 smithi119 ceph-mon[21835]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-04-22T15:09:30.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:30 smithi119 ceph-mon[21835]: osdmap e46: 7 total, 6 up, 7 in 2024-04-22T15:09:30.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:30 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:30.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:30 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:30.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:30 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:31.512 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:31 smithi186 ceph-mon[24041]: purged_snaps scrub starts 2024-04-22T15:09:31.513 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:31 smithi186 ceph-mon[24041]: purged_snaps scrub ok 2024-04-22T15:09:31.513 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:31 smithi186 ceph-mon[24041]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:31.513 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:31 smithi186 ceph-mon[24041]: osd.6 [v2:172.21.15.186:6816/2393038,v1:172.21.15.186:6817/2393038] boot 2024-04-22T15:09:31.513 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:31 smithi186 ceph-mon[24041]: osdmap e47: 7 total, 7 up, 7 in 2024-04-22T15:09:31.513 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:31 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:31.513 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:31 smithi186 ceph-mon[24041]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T15:09:31.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:31 smithi119 ceph-mon[21835]: purged_snaps scrub starts 2024-04-22T15:09:31.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:31 smithi119 ceph-mon[21835]: purged_snaps scrub ok 2024-04-22T15:09:31.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:31 smithi119 ceph-mon[21835]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:31.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:31 smithi119 ceph-mon[21835]: osd.6 [v2:172.21.15.186:6816/2393038,v1:172.21.15.186:6817/2393038] boot 2024-04-22T15:09:31.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:31 smithi119 ceph-mon[21835]: osdmap e47: 7 total, 7 up, 7 in 2024-04-22T15:09:31.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:31 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:31.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:31 smithi119 ceph-mon[21835]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T15:09:32.504 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi186:vg_nvme/lv_1 2024-04-22T15:09:33.652 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:33 smithi186 ceph-mon[24041]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:33.652 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:33 smithi186 ceph-mon[24041]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T15:09:33.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:33 smithi119 ceph-mon[21835]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:33.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:33 smithi119 ceph-mon[21835]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T15:09:34.666 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:34 smithi186 ceph-mon[24041]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T15:09:34.666 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:34.666 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:34.666 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:34.666 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:34.666 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:34.667 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:34.667 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:34.667 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:34.667 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:34.667 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:34 smithi119 ceph-mon[21835]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T15:09:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:34.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:34.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:34.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:35.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:35 smithi186 ceph-mon[24041]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:35.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:35 smithi186 ceph-mon[24041]: Detected new or changed devices on smithi186 2024-04-22T15:09:35.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:35 smithi186 ceph-mon[24041]: Adjusting osd_memory_target on smithi186 to 5312M 2024-04-22T15:09:35.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:35 smithi186 ceph-mon[24041]: osdmap e51: 7 total, 7 up, 7 in 2024-04-22T15:09:35.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:09:35.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:09:35.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:35.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:35 smithi119 ceph-mon[21835]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:35.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:35 smithi119 ceph-mon[21835]: Detected new or changed devices on smithi186 2024-04-22T15:09:35.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:35 smithi119 ceph-mon[21835]: Adjusting osd_memory_target on smithi186 to 5312M 2024-04-22T15:09:35.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:35 smithi119 ceph-mon[21835]: osdmap e51: 7 total, 7 up, 7 in 2024-04-22T15:09:35.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:09:35.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:09:35.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:36.743 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:36 smithi186 ceph-mon[24041]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:36.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:36 smithi119 ceph-mon[21835]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:37 smithi186 ceph-mon[24041]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-04-22T15:09:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:37 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/3064972151' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbc9e890-66ed-425b-bddb-4d1bfd2d7ac1"}]: dispatch 2024-04-22T15:09:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:37 smithi186 ceph-mon[24041]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbc9e890-66ed-425b-bddb-4d1bfd2d7ac1"}]: dispatch 2024-04-22T15:09:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:37 smithi186 ceph-mon[24041]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbc9e890-66ed-425b-bddb-4d1bfd2d7ac1"}]': finished 2024-04-22T15:09:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:37 smithi186 ceph-mon[24041]: osdmap e52: 8 total, 7 up, 8 in 2024-04-22T15:09:37.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:37.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:37 smithi119 ceph-mon[21835]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-04-22T15:09:37.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:37 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/3064972151' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbc9e890-66ed-425b-bddb-4d1bfd2d7ac1"}]: dispatch 2024-04-22T15:09:37.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:37 smithi119 ceph-mon[21835]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbc9e890-66ed-425b-bddb-4d1bfd2d7ac1"}]: dispatch 2024-04-22T15:09:37.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:37 smithi119 ceph-mon[21835]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbc9e890-66ed-425b-bddb-4d1bfd2d7ac1"}]': finished 2024-04-22T15:09:37.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:37 smithi119 ceph-mon[21835]: osdmap e52: 8 total, 7 up, 8 in 2024-04-22T15:09:37.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:38.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:38 smithi119 ceph-mon[21835]: from='client.? 172.21.15.186:0/2278922888' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:09:39.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:38 smithi186 ceph-mon[24041]: from='client.? 172.21.15.186:0/2278922888' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:09:39.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:39 smithi119 ceph-mon[21835]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-04-22T15:09:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:39 smithi186 ceph-mon[24041]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-04-22T15:09:41.636 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:41 smithi186 ceph-mon[24041]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-22T15:09:41.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:41 smithi119 ceph-mon[21835]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-22T15:09:43.713 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:43 smithi186 ceph-mon[24041]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T15:09:43.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:43 smithi119 ceph-mon[21835]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T15:09:44.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:44 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:09:44.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:44 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:44.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:44 smithi186 ceph-mon[24041]: Deploying daemon osd.7 on smithi186 2024-04-22T15:09:44.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:44 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:09:44.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:44 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:44.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:44 smithi119 ceph-mon[21835]: Deploying daemon osd.7 on smithi186 2024-04-22T15:09:45.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:45 smithi186 ceph-mon[24041]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-22T15:09:45.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:45 smithi119 ceph-mon[21835]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-22T15:09:47.761 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:47 smithi186 ceph-mon[24041]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:47.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:47 smithi119 ceph-mon[21835]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:49.634 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:49 smithi186 ceph-mon[24041]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:49.634 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:49 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:49.634 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:49 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:49.634 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:49 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:49.635 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:49 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:49.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:49 smithi119 ceph-mon[21835]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:49.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:49 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:49.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:49 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:49.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:49 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:49.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:49 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:50.958 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:09:50.958 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:09:50.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:09:50.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:09:51.691 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:51 smithi186 ceph-mon[24041]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:51.701 INFO:teuthology.orchestra.run.smithi186.stdout:Created osd(s) 7 on host 'smithi186' 2024-04-22T15:09:51.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:51 smithi119 ceph-mon[21835]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:52.556 DEBUG:teuthology.orchestra.run.smithi186:osd.7> sudo journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.7.service 2024-04-22T15:09:52.559 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-22T15:09:52.560 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-22T15:09:52.874 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:52 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:52.874 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:52 smithi119 ceph-mon[21835]: from='osd.7 [v2:172.21.15.186:6824/4138513024,v1:172.21.15.186:6825/4138513024]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:09:52.874 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:52 smithi119 ceph-mon[21835]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:09:52.894 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:52 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:52.894 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:52 smithi186 ceph-mon[24041]: from='osd.7 [v2:172.21.15.186:6824/4138513024,v1:172.21.15.186:6825/4138513024]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:09:52.894 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:52 smithi186 ceph-mon[24041]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:09:53.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:53 smithi186 ceph-mon[24041]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:53.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:53 smithi186 ceph-mon[24041]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T15:09:53.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:53 smithi186 ceph-mon[24041]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T15:09:53.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:53 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:53.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:53 smithi186 ceph-mon[24041]: from='osd.7 [v2:172.21.15.186:6824/4138513024,v1:172.21.15.186:6825/4138513024]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:09:53.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:53 smithi186 ceph-mon[24041]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:09:53.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:53 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:53.776 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:09:53 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[37219]: 2024-04-22T15:09:53.706+0000 7f3540685700 -1 osd.7 0 waiting for initial osdmap 2024-04-22T15:09:53.776 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:09:53 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[37219]: 2024-04-22T15:09:53.715+0000 7f353b01c700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:09:53.977 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:53 smithi119 ceph-mon[21835]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:53.978 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:53 smithi119 ceph-mon[21835]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T15:09:53.978 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:53 smithi119 ceph-mon[21835]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T15:09:53.978 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:53 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:53.978 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:53 smithi119 ceph-mon[21835]: from='osd.7 [v2:172.21.15.186:6824/4138513024,v1:172.21.15.186:6825/4138513024]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:09:53.978 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:53 smithi119 ceph-mon[21835]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:09:53.978 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:53 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:54.639 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:09:54.942 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:54 smithi119 ceph-mon[21835]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-04-22T15:09:54.942 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:54 smithi119 ceph-mon[21835]: osdmap e54: 8 total, 7 up, 8 in 2024-04-22T15:09:54.942 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:54 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:54.942 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:54 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:54.942 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:54 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3357278909' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T15:09:55.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:54 smithi186 ceph-mon[24041]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-04-22T15:09:55.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:54 smithi186 ceph-mon[24041]: osdmap e54: 8 total, 7 up, 8 in 2024-04-22T15:09:55.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:54 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:55.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:54 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:55.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:54 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/3357278909' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T15:09:55.334 INFO:teuthology.orchestra.run.smithi119.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1713798570,"num_in_osds":8,"osd_in_since":1713798577,"num_remapped_pgs":0} 2024-04-22T15:09:55.973 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:55 smithi186 ceph-mon[24041]: purged_snaps scrub starts 2024-04-22T15:09:55.973 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:55 smithi186 ceph-mon[24041]: purged_snaps scrub ok 2024-04-22T15:09:55.973 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:55 smithi186 ceph-mon[24041]: pgmap v164: 1 pgs: 1 peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:55.973 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:55 smithi186 ceph-mon[24041]: osd.7 [v2:172.21.15.186:6824/4138513024,v1:172.21.15.186:6825/4138513024] boot 2024-04-22T15:09:55.973 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:55 smithi186 ceph-mon[24041]: osdmap e55: 8 total, 8 up, 8 in 2024-04-22T15:09:55.973 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:55 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:55.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:55 smithi119 ceph-mon[21835]: purged_snaps scrub starts 2024-04-22T15:09:55.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:55 smithi119 ceph-mon[21835]: purged_snaps scrub ok 2024-04-22T15:09:55.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:55 smithi119 ceph-mon[21835]: pgmap v164: 1 pgs: 1 peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:55.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:55 smithi119 ceph-mon[21835]: osd.7 [v2:172.21.15.186:6824/4138513024,v1:172.21.15.186:6825/4138513024] boot 2024-04-22T15:09:55.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:55 smithi119 ceph-mon[21835]: osdmap e55: 8 total, 8 up, 8 in 2024-04-22T15:09:55.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:55 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:56.335 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-22T15:09:56.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T15:09:56.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:56.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:56.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:56.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:56.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:56.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:56.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:56.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:56.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:56.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:56.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:56.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:56 smithi119 ceph-mon[21835]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T15:09:57.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T15:09:57.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:57.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:57.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:57.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:57.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:57.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:57.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:57.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:57.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:57.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:57.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:09:57.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:56 smithi186 ceph-mon[24041]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T15:09:57.970 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:57 smithi119 ceph-mon[21835]: pgmap v167: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:57.970 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:57 smithi119 ceph-mon[21835]: Detected new or changed devices on smithi186 2024-04-22T15:09:57.970 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:57 smithi119 ceph-mon[21835]: Adjusting osd_memory_target on smithi186 to 3984M 2024-04-22T15:09:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:57 smithi186 ceph-mon[24041]: pgmap v167: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:57 smithi186 ceph-mon[24041]: Detected new or changed devices on smithi186 2024-04-22T15:09:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:57 smithi186 ceph-mon[24041]: Adjusting osd_memory_target on smithi186 to 3984M 2024-04-22T15:09:58.368 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:09:58.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:58 smithi119 ceph-mon[21835]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T15:09:58.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:58 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/1203494323' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T15:09:59.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:58 smithi186 ceph-mon[24041]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T15:09:59.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:58 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/1203494323' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T15:09:59.083 INFO:teuthology.orchestra.run.smithi119.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1713798594,"num_in_osds":8,"osd_in_since":1713798577,"num_remapped_pgs":0} 2024-04-22T15:09:59.083 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T15:09:59.293 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:09:59.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:59 smithi119 ceph-mon[21835]: pgmap v170: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:59.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:09:59 smithi119 ceph-mon[21835]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T15:10:00.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:59 smithi186 ceph-mon[24041]: pgmap v170: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:00.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:09:59 smithi186 ceph-mon[24041]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T15:10:00.755 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:00 smithi119 ceph-mon[21835]: overall HEALTH_OK 2024-04-22T15:10:01.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:00 smithi186 ceph-mon[24041]: overall HEALTH_OK 2024-04-22T15:10:01.155 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:10:01.155 INFO:teuthology.orchestra.run.smithi119.stdout:{"epoch":59,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","created":"2024-04-22T15:03:56.133758+0000","modified":"2024-04-22T15:09:58.717667+0000","last_up_change":"2024-04-22T15:09:54.700788+0000","last_in_change":"2024-04-22T15:09:37.476174+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T15:07:56.362330+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"5e297b79-9f0a-4ca7-b0c1-bbb13a534936","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":1289753238},{"type":"v1","addr":"172.21.15.119:6803","nonce":1289753238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6804","nonce":1289753238},{"type":"v1","addr":"172.21.15.119:6805","nonce":1289753238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":1289753238},{"type":"v1","addr":"172.21.15.119:6809","nonce":1289753238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":1289753238},{"type":"v1","addr":"172.21.15.119:6807","nonce":1289753238}]},"public_addr":"172.21.15.119:6803/1289753238","cluster_addr":"172.21.15.119:6805/1289753238","heartbeat_back_addr":"172.21.15.119:6809/1289753238","heartbeat_front_addr":"172.21.15.119:6807/1289753238","state":["exists","up"]},{"osd":1,"uuid":"95c40890-6713-4e2a-94f3-9a2adb44dc20","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6810","nonce":141206705},{"type":"v1","addr":"172.21.15.119:6811","nonce":141206705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6812","nonce":141206705},{"type":"v1","addr":"172.21.15.119:6813","nonce":141206705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6816","nonce":141206705},{"type":"v1","addr":"172.21.15.119:6817","nonce":141206705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6814","nonce":141206705},{"type":"v1","addr":"172.21.15.119:6815","nonce":141206705}]},"public_addr":"172.21.15.119:6811/141206705","cluster_addr":"172.21.15.119:6813/141206705","heartbeat_back_addr":"172.21.15.119:6817/141206705","heartbeat_front_addr":"172.21.15.119:6815/141206705","state":["exists","up"]},{"osd":2,"uuid":"36c6bc11-2dd0-48f5-b284-73b17aaeea69","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6818","nonce":1451810811},{"type":"v1","addr":"172.21.15.119:6819","nonce":1451810811}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6820","nonce":1451810811},{"type":"v1","addr":"172.21.15.119:6821","nonce":1451810811}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6824","nonce":1451810811},{"type":"v1","addr":"172.21.15.119:6825","nonce":1451810811}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6822","nonce":1451810811},{"type":"v1","addr":"172.21.15.119:6823","nonce":1451810811}]},"public_addr":"172.21.15.119:6819/1451810811","cluster_addr":"172.21.15.119:6821/1451810811","heartbeat_back_addr":"172.21.15.119:6825/1451810811","heartbeat_front_addr":"172.21.15.119:6823/1451810811","state":["exists","up"]},{"osd":3,"uuid":"e3c166b6-455e-4900-ae71-472de8a7fdb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6826","nonce":602651652},{"type":"v1","addr":"172.21.15.119:6827","nonce":602651652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6828","nonce":602651652},{"type":"v1","addr":"172.21.15.119:6829","nonce":602651652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6832","nonce":602651652},{"type":"v1","addr":"172.21.15.119:6833","nonce":602651652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6830","nonce":602651652},{"type":"v1","addr":"172.21.15.119:6831","nonce":602651652}]},"public_addr":"172.21.15.119:6827/602651652","cluster_addr":"172.21.15.119:6829/602651652","heartbeat_back_addr":"172.21.15.119:6833/602651652","heartbeat_front_addr":"172.21.15.119:6831/602651652","state":["exists","up"]},{"osd":4,"uuid":"a7de45c9-0187-4e07-8980-90e599176928","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6800","nonce":3140332101},{"type":"v1","addr":"172.21.15.186:6801","nonce":3140332101}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6802","nonce":3140332101},{"type":"v1","addr":"172.21.15.186:6803","nonce":3140332101}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6806","nonce":3140332101},{"type":"v1","addr":"172.21.15.186:6807","nonce":3140332101}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6804","nonce":3140332101},{"type":"v1","addr":"172.21.15.186:6805","nonce":3140332101}]},"public_addr":"172.21.15.186:6801/3140332101","cluster_addr":"172.21.15.186:6803/3140332101","heartbeat_back_addr":"172.21.15.186:6807/3140332101","heartbeat_front_addr":"172.21.15.186:6805/3140332101","state":["exists","up"]},{"osd":5,"uuid":"ea7cee92-1fa1-4f5b-9b42-ca5673853759","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6808","nonce":3541622350},{"type":"v1","addr":"172.21.15.186:6809","nonce":3541622350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6810","nonce":3541622350},{"type":"v1","addr":"172.21.15.186:6811","nonce":3541622350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6814","nonce":3541622350},{"type":"v1","addr":"172.21.15.186:6815","nonce":3541622350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6812","nonce":3541622350},{"type":"v1","addr":"172.21.15.186:6813","nonce":3541622350}]},"public_addr":"172.21.15.186:6809/3541622350","cluster_addr":"172.21.15.186:6811/3541622350","heartbeat_back_addr":"172.21.15.186:6815/3541622350","heartbeat_front_addr":"172.21.15.186:6813/3541622350","state":["exists","up"]},{"osd":6,"uuid":"4ea83fed-ee69-4db5-9ed9-9912da48ff93","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6816","nonce":2393038},{"type":"v1","addr":"172.21.15.186:6817","nonce":2393038}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6818","nonce":2393038},{"type":"v1","addr":"172.21.15.186:6819","nonce":2393038}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6822","nonce":2393038},{"type":"v1","addr":"172.21.15.186:6823","nonce":2393038}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6820","nonce":2393038},{"type":"v1","addr":"172.21.15.186:6821","nonce":2393038}]},"public_addr":"172.21.15.186:6817/2393038","cluster_addr":"172.21.15.186:6819/2393038","heartbeat_back_addr":"172.21.15.186:6823/2393038","heartbeat_front_addr":"172.21.15.186:6821/2393038","state":["exists","up"]},{"osd":7,"uuid":"cbc9e890-66ed-425b-bddb-4d1bfd2d7ac1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6824","nonce":4138513024},{"type":"v1","addr":"172.21.15.186:6825","nonce":4138513024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6826","nonce":4138513024},{"type":"v1","addr":"172.21.15.186:6827","nonce":4138513024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6830","nonce":4138513024},{"type":"v1","addr":"172.21.15.186:6831","nonce":4138513024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6828","nonce":4138513024},{"type":"v1","addr":"172.21.15.186:6829","nonce":4138513024}]},"public_addr":"172.21.15.186:6825/4138513024","cluster_addr":"172.21.15.186:6827/4138513024","heartbeat_back_addr":"172.21.15.186:6831/4138513024","heartbeat_front_addr":"172.21.15.186:6829/4138513024","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:06:56.737785+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:27.826378+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:53.696435+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:19.937871+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:41.042875+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:09:04.944165+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:09:28.395792+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:09:53.033643+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.119:0/245853678":"2024-04-23T15:05:49.673673+0000","172.21.15.119:6800/3850519770":"2024-04-23T15:05:49.673673+0000","172.21.15.119:0/2943930675":"2024-04-23T15:05:49.673673+0000","172.21.15.119:6801/3850519770":"2024-04-23T15:05:49.673673+0000","172.21.15.119:6800/2044153072":"2024-04-23T15:04:54.754937+0000","172.21.15.119:0/1796407705":"2024-04-23T15:04:54.754937+0000","172.21.15.119:0/3054130325":"2024-04-23T15:04:54.754937+0000","172.21.15.119:0/2066724923":"2024-04-23T15:04:54.754937+0000","172.21.15.119:0/4092556581":"2024-04-23T15:04:24.970842+0000","172.21.15.119:0/3267006356":"2024-04-23T15:04:24.970842+0000","172.21.15.119:0/3417819620":"2024-04-23T15:04:24.970842+0000","172.21.15.119:6801/2044153072":"2024-04-23T15:04:54.754937+0000","172.21.15.119:6801/3215898620":"2024-04-23T15:04:24.970842+0000","172.21.15.119:0/3856353948":"2024-04-23T15:05:49.673673+0000","172.21.15.119:6800/3215898620":"2024-04-23T15:04:24.970842+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T15:10:01.880 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-22T15:07:56.362330+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-22T15:10:01.880 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-22T15:10:01.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:01 smithi119 ceph-mon[21835]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-04-22T15:10:01.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:01 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2981984853' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:10:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:01 smithi186 ceph-mon[24041]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-04-22T15:10:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:01 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2981984853' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:10:02.092 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:03.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:03 smithi119 ceph-mon[21835]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T15:10:03.907 INFO:teuthology.orchestra.run.smithi119.stdout:pg_num: 1 2024-04-22T15:10:04.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:03 smithi186 ceph-mon[24041]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T15:10:04.702 INFO:tasks.cephadm:Setting up client nodes... 2024-04-22T15:10:04.702 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-22T15:10:04.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:04 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/1526351334' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T15:10:05.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:04 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/1526351334' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T15:10:05.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:05 smithi119 ceph-mon[21835]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-22T15:10:06.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:05 smithi186 ceph-mon[24041]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-22T15:10:06.758 INFO:teuthology.orchestra.run.smithi119.stdout:[client.0] 2024-04-22T15:10:06.759 INFO:teuthology.orchestra.run.smithi119.stdout: key = AQDOfSZmFwPYLBAAN7y6Kq8dN0jmZRHV2bySjQ== 2024-04-22T15:10:07.465 DEBUG:teuthology.orchestra.run.smithi119:> set -ex 2024-04-22T15:10:07.466 DEBUG:teuthology.orchestra.run.smithi119:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-22T15:10:07.466 DEBUG:teuthology.orchestra.run.smithi119:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-22T15:10:07.501 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-22T15:10:07.501 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-22T15:10:07.502 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-22T15:10:07.749 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:07.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:07 smithi119 ceph-mon[21835]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-22T15:10:07.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:07 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/12390385' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T15:10:07.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:07 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/12390385' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T15:10:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:07 smithi186 ceph-mon[24041]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-22T15:10:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:07 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/12390385' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T15:10:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:07 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/12390385' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T15:10:09.613 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:10:09.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:09 smithi119 ceph-mon[21835]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:10:09.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:09 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/1001188962' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T15:10:10.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:09 smithi186 ceph-mon[24041]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:10:10.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:09 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/1001188962' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T15:10:10.382 INFO:teuthology.orchestra.run.smithi119.stdout:{"epoch":19,"active_gid":14204,"active_name":"smithi119.jjbxxd","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":2410827406},{"type":"v1","addr":"172.21.15.119:6801","nonce":2410827406}]},"active_addr":"172.21.15.119:6801/2410827406","active_change":"2024-04-22T15:05:49.673982+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14222,"name":"smithi186.vtjgwg","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.119:8443/","prometheus":"http://172.21.15.119:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.119:0","nonce":2360565605}]},{"addrvec":[{"type":"v2","addr":"172.21.15.119:0","nonce":2426210089}]},{"addrvec":[{"type":"v2","addr":"172.21.15.119:0","nonce":3510466931}]},{"addrvec":[{"type":"v2","addr":"172.21.15.119:0","nonce":2474582802}]}]}} 2024-04-22T15:10:10.388 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-22T15:10:10.388 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-22T15:10:10.389 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T15:10:10.602 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:11.882 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:11 smithi119 ceph-mon[21835]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s, 0 objects/s recovering 2024-04-22T15:10:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:11 smithi186 ceph-mon[24041]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s, 0 objects/s recovering 2024-04-22T15:10:12.255 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:10:12.255 INFO:teuthology.orchestra.run.smithi119.stdout:{"epoch":59,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","created":"2024-04-22T15:03:56.133758+0000","modified":"2024-04-22T15:09:58.717667+0000","last_up_change":"2024-04-22T15:09:54.700788+0000","last_in_change":"2024-04-22T15:09:37.476174+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T15:07:56.362330+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"5e297b79-9f0a-4ca7-b0c1-bbb13a534936","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":1289753238},{"type":"v1","addr":"172.21.15.119:6803","nonce":1289753238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6804","nonce":1289753238},{"type":"v1","addr":"172.21.15.119:6805","nonce":1289753238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":1289753238},{"type":"v1","addr":"172.21.15.119:6809","nonce":1289753238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":1289753238},{"type":"v1","addr":"172.21.15.119:6807","nonce":1289753238}]},"public_addr":"172.21.15.119:6803/1289753238","cluster_addr":"172.21.15.119:6805/1289753238","heartbeat_back_addr":"172.21.15.119:6809/1289753238","heartbeat_front_addr":"172.21.15.119:6807/1289753238","state":["exists","up"]},{"osd":1,"uuid":"95c40890-6713-4e2a-94f3-9a2adb44dc20","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6810","nonce":141206705},{"type":"v1","addr":"172.21.15.119:6811","nonce":141206705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6812","nonce":141206705},{"type":"v1","addr":"172.21.15.119:6813","nonce":141206705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6816","nonce":141206705},{"type":"v1","addr":"172.21.15.119:6817","nonce":141206705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6814","nonce":141206705},{"type":"v1","addr":"172.21.15.119:6815","nonce":141206705}]},"public_addr":"172.21.15.119:6811/141206705","cluster_addr":"172.21.15.119:6813/141206705","heartbeat_back_addr":"172.21.15.119:6817/141206705","heartbeat_front_addr":"172.21.15.119:6815/141206705","state":["exists","up"]},{"osd":2,"uuid":"36c6bc11-2dd0-48f5-b284-73b17aaeea69","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6818","nonce":1451810811},{"type":"v1","addr":"172.21.15.119:6819","nonce":1451810811}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6820","nonce":1451810811},{"type":"v1","addr":"172.21.15.119:6821","nonce":1451810811}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6824","nonce":1451810811},{"type":"v1","addr":"172.21.15.119:6825","nonce":1451810811}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6822","nonce":1451810811},{"type":"v1","addr":"172.21.15.119:6823","nonce":1451810811}]},"public_addr":"172.21.15.119:6819/1451810811","cluster_addr":"172.21.15.119:6821/1451810811","heartbeat_back_addr":"172.21.15.119:6825/1451810811","heartbeat_front_addr":"172.21.15.119:6823/1451810811","state":["exists","up"]},{"osd":3,"uuid":"e3c166b6-455e-4900-ae71-472de8a7fdb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6826","nonce":602651652},{"type":"v1","addr":"172.21.15.119:6827","nonce":602651652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6828","nonce":602651652},{"type":"v1","addr":"172.21.15.119:6829","nonce":602651652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6832","nonce":602651652},{"type":"v1","addr":"172.21.15.119:6833","nonce":602651652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6830","nonce":602651652},{"type":"v1","addr":"172.21.15.119:6831","nonce":602651652}]},"public_addr":"172.21.15.119:6827/602651652","cluster_addr":"172.21.15.119:6829/602651652","heartbeat_back_addr":"172.21.15.119:6833/602651652","heartbeat_front_addr":"172.21.15.119:6831/602651652","state":["exists","up"]},{"osd":4,"uuid":"a7de45c9-0187-4e07-8980-90e599176928","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6800","nonce":3140332101},{"type":"v1","addr":"172.21.15.186:6801","nonce":3140332101}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6802","nonce":3140332101},{"type":"v1","addr":"172.21.15.186:6803","nonce":3140332101}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6806","nonce":3140332101},{"type":"v1","addr":"172.21.15.186:6807","nonce":3140332101}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6804","nonce":3140332101},{"type":"v1","addr":"172.21.15.186:6805","nonce":3140332101}]},"public_addr":"172.21.15.186:6801/3140332101","cluster_addr":"172.21.15.186:6803/3140332101","heartbeat_back_addr":"172.21.15.186:6807/3140332101","heartbeat_front_addr":"172.21.15.186:6805/3140332101","state":["exists","up"]},{"osd":5,"uuid":"ea7cee92-1fa1-4f5b-9b42-ca5673853759","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6808","nonce":3541622350},{"type":"v1","addr":"172.21.15.186:6809","nonce":3541622350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6810","nonce":3541622350},{"type":"v1","addr":"172.21.15.186:6811","nonce":3541622350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6814","nonce":3541622350},{"type":"v1","addr":"172.21.15.186:6815","nonce":3541622350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6812","nonce":3541622350},{"type":"v1","addr":"172.21.15.186:6813","nonce":3541622350}]},"public_addr":"172.21.15.186:6809/3541622350","cluster_addr":"172.21.15.186:6811/3541622350","heartbeat_back_addr":"172.21.15.186:6815/3541622350","heartbeat_front_addr":"172.21.15.186:6813/3541622350","state":["exists","up"]},{"osd":6,"uuid":"4ea83fed-ee69-4db5-9ed9-9912da48ff93","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6816","nonce":2393038},{"type":"v1","addr":"172.21.15.186:6817","nonce":2393038}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6818","nonce":2393038},{"type":"v1","addr":"172.21.15.186:6819","nonce":2393038}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6822","nonce":2393038},{"type":"v1","addr":"172.21.15.186:6823","nonce":2393038}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6820","nonce":2393038},{"type":"v1","addr":"172.21.15.186:6821","nonce":2393038}]},"public_addr":"172.21.15.186:6817/2393038","cluster_addr":"172.21.15.186:6819/2393038","heartbeat_back_addr":"172.21.15.186:6823/2393038","heartbeat_front_addr":"172.21.15.186:6821/2393038","state":["exists","up"]},{"osd":7,"uuid":"cbc9e890-66ed-425b-bddb-4d1bfd2d7ac1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6824","nonce":4138513024},{"type":"v1","addr":"172.21.15.186:6825","nonce":4138513024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6826","nonce":4138513024},{"type":"v1","addr":"172.21.15.186:6827","nonce":4138513024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6830","nonce":4138513024},{"type":"v1","addr":"172.21.15.186:6831","nonce":4138513024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6828","nonce":4138513024},{"type":"v1","addr":"172.21.15.186:6829","nonce":4138513024}]},"public_addr":"172.21.15.186:6825/4138513024","cluster_addr":"172.21.15.186:6827/4138513024","heartbeat_back_addr":"172.21.15.186:6831/4138513024","heartbeat_front_addr":"172.21.15.186:6829/4138513024","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:06:56.737785+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:27.826378+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:53.696435+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:19.937871+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:41.042875+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:09:04.944165+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:09:28.395792+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:09:53.033643+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.119:0/245853678":"2024-04-23T15:05:49.673673+0000","172.21.15.119:6800/3850519770":"2024-04-23T15:05:49.673673+0000","172.21.15.119:0/2943930675":"2024-04-23T15:05:49.673673+0000","172.21.15.119:6801/3850519770":"2024-04-23T15:05:49.673673+0000","172.21.15.119:6800/2044153072":"2024-04-23T15:04:54.754937+0000","172.21.15.119:0/1796407705":"2024-04-23T15:04:54.754937+0000","172.21.15.119:0/3054130325":"2024-04-23T15:04:54.754937+0000","172.21.15.119:0/2066724923":"2024-04-23T15:04:54.754937+0000","172.21.15.119:0/4092556581":"2024-04-23T15:04:24.970842+0000","172.21.15.119:0/3267006356":"2024-04-23T15:04:24.970842+0000","172.21.15.119:0/3417819620":"2024-04-23T15:04:24.970842+0000","172.21.15.119:6801/2044153072":"2024-04-23T15:04:54.754937+0000","172.21.15.119:6801/3215898620":"2024-04-23T15:04:24.970842+0000","172.21.15.119:0/3856353948":"2024-04-23T15:05:49.673673+0000","172.21.15.119:6800/3215898620":"2024-04-23T15:04:24.970842+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T15:10:12.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:12 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2477015870' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:10:12.978 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-22T15:10:12.979 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T15:10:13.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:12 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2477015870' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:10:13.191 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:13.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:13 smithi119 ceph-mon[21835]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:13 smithi186 ceph-mon[24041]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:14.804 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:14 smithi119 ceph-mon[21835]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:14.978 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:10:14.978 INFO:teuthology.orchestra.run.smithi119.stdout:{"epoch":59,"fsid":"74739db6-00b9-11ef-bc93-c7b262605968","created":"2024-04-22T15:03:56.133758+0000","modified":"2024-04-22T15:09:58.717667+0000","last_up_change":"2024-04-22T15:09:54.700788+0000","last_in_change":"2024-04-22T15:09:37.476174+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T15:07:56.362330+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"5e297b79-9f0a-4ca7-b0c1-bbb13a534936","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":1289753238},{"type":"v1","addr":"172.21.15.119:6803","nonce":1289753238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6804","nonce":1289753238},{"type":"v1","addr":"172.21.15.119:6805","nonce":1289753238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":1289753238},{"type":"v1","addr":"172.21.15.119:6809","nonce":1289753238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":1289753238},{"type":"v1","addr":"172.21.15.119:6807","nonce":1289753238}]},"public_addr":"172.21.15.119:6803/1289753238","cluster_addr":"172.21.15.119:6805/1289753238","heartbeat_back_addr":"172.21.15.119:6809/1289753238","heartbeat_front_addr":"172.21.15.119:6807/1289753238","state":["exists","up"]},{"osd":1,"uuid":"95c40890-6713-4e2a-94f3-9a2adb44dc20","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6810","nonce":141206705},{"type":"v1","addr":"172.21.15.119:6811","nonce":141206705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6812","nonce":141206705},{"type":"v1","addr":"172.21.15.119:6813","nonce":141206705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6816","nonce":141206705},{"type":"v1","addr":"172.21.15.119:6817","nonce":141206705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6814","nonce":141206705},{"type":"v1","addr":"172.21.15.119:6815","nonce":141206705}]},"public_addr":"172.21.15.119:6811/141206705","cluster_addr":"172.21.15.119:6813/141206705","heartbeat_back_addr":"172.21.15.119:6817/141206705","heartbeat_front_addr":"172.21.15.119:6815/141206705","state":["exists","up"]},{"osd":2,"uuid":"36c6bc11-2dd0-48f5-b284-73b17aaeea69","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6818","nonce":1451810811},{"type":"v1","addr":"172.21.15.119:6819","nonce":1451810811}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6820","nonce":1451810811},{"type":"v1","addr":"172.21.15.119:6821","nonce":1451810811}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6824","nonce":1451810811},{"type":"v1","addr":"172.21.15.119:6825","nonce":1451810811}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6822","nonce":1451810811},{"type":"v1","addr":"172.21.15.119:6823","nonce":1451810811}]},"public_addr":"172.21.15.119:6819/1451810811","cluster_addr":"172.21.15.119:6821/1451810811","heartbeat_back_addr":"172.21.15.119:6825/1451810811","heartbeat_front_addr":"172.21.15.119:6823/1451810811","state":["exists","up"]},{"osd":3,"uuid":"e3c166b6-455e-4900-ae71-472de8a7fdb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6826","nonce":602651652},{"type":"v1","addr":"172.21.15.119:6827","nonce":602651652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6828","nonce":602651652},{"type":"v1","addr":"172.21.15.119:6829","nonce":602651652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6832","nonce":602651652},{"type":"v1","addr":"172.21.15.119:6833","nonce":602651652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6830","nonce":602651652},{"type":"v1","addr":"172.21.15.119:6831","nonce":602651652}]},"public_addr":"172.21.15.119:6827/602651652","cluster_addr":"172.21.15.119:6829/602651652","heartbeat_back_addr":"172.21.15.119:6833/602651652","heartbeat_front_addr":"172.21.15.119:6831/602651652","state":["exists","up"]},{"osd":4,"uuid":"a7de45c9-0187-4e07-8980-90e599176928","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6800","nonce":3140332101},{"type":"v1","addr":"172.21.15.186:6801","nonce":3140332101}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6802","nonce":3140332101},{"type":"v1","addr":"172.21.15.186:6803","nonce":3140332101}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6806","nonce":3140332101},{"type":"v1","addr":"172.21.15.186:6807","nonce":3140332101}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6804","nonce":3140332101},{"type":"v1","addr":"172.21.15.186:6805","nonce":3140332101}]},"public_addr":"172.21.15.186:6801/3140332101","cluster_addr":"172.21.15.186:6803/3140332101","heartbeat_back_addr":"172.21.15.186:6807/3140332101","heartbeat_front_addr":"172.21.15.186:6805/3140332101","state":["exists","up"]},{"osd":5,"uuid":"ea7cee92-1fa1-4f5b-9b42-ca5673853759","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6808","nonce":3541622350},{"type":"v1","addr":"172.21.15.186:6809","nonce":3541622350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6810","nonce":3541622350},{"type":"v1","addr":"172.21.15.186:6811","nonce":3541622350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6814","nonce":3541622350},{"type":"v1","addr":"172.21.15.186:6815","nonce":3541622350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6812","nonce":3541622350},{"type":"v1","addr":"172.21.15.186:6813","nonce":3541622350}]},"public_addr":"172.21.15.186:6809/3541622350","cluster_addr":"172.21.15.186:6811/3541622350","heartbeat_back_addr":"172.21.15.186:6815/3541622350","heartbeat_front_addr":"172.21.15.186:6813/3541622350","state":["exists","up"]},{"osd":6,"uuid":"4ea83fed-ee69-4db5-9ed9-9912da48ff93","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6816","nonce":2393038},{"type":"v1","addr":"172.21.15.186:6817","nonce":2393038}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6818","nonce":2393038},{"type":"v1","addr":"172.21.15.186:6819","nonce":2393038}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6822","nonce":2393038},{"type":"v1","addr":"172.21.15.186:6823","nonce":2393038}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6820","nonce":2393038},{"type":"v1","addr":"172.21.15.186:6821","nonce":2393038}]},"public_addr":"172.21.15.186:6817/2393038","cluster_addr":"172.21.15.186:6819/2393038","heartbeat_back_addr":"172.21.15.186:6823/2393038","heartbeat_front_addr":"172.21.15.186:6821/2393038","state":["exists","up"]},{"osd":7,"uuid":"cbc9e890-66ed-425b-bddb-4d1bfd2d7ac1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6824","nonce":4138513024},{"type":"v1","addr":"172.21.15.186:6825","nonce":4138513024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6826","nonce":4138513024},{"type":"v1","addr":"172.21.15.186:6827","nonce":4138513024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6830","nonce":4138513024},{"type":"v1","addr":"172.21.15.186:6831","nonce":4138513024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6828","nonce":4138513024},{"type":"v1","addr":"172.21.15.186:6829","nonce":4138513024}]},"public_addr":"172.21.15.186:6825/4138513024","cluster_addr":"172.21.15.186:6827/4138513024","heartbeat_back_addr":"172.21.15.186:6831/4138513024","heartbeat_front_addr":"172.21.15.186:6829/4138513024","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:06:56.737785+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:27.826378+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:53.696435+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:19.937871+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:41.042875+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:09:04.944165+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:09:28.395792+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:09:53.033643+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.119:0/245853678":"2024-04-23T15:05:49.673673+0000","172.21.15.119:6800/3850519770":"2024-04-23T15:05:49.673673+0000","172.21.15.119:0/2943930675":"2024-04-23T15:05:49.673673+0000","172.21.15.119:6801/3850519770":"2024-04-23T15:05:49.673673+0000","172.21.15.119:6800/2044153072":"2024-04-23T15:04:54.754937+0000","172.21.15.119:0/1796407705":"2024-04-23T15:04:54.754937+0000","172.21.15.119:0/3054130325":"2024-04-23T15:04:54.754937+0000","172.21.15.119:0/2066724923":"2024-04-23T15:04:54.754937+0000","172.21.15.119:0/4092556581":"2024-04-23T15:04:24.970842+0000","172.21.15.119:0/3267006356":"2024-04-23T15:04:24.970842+0000","172.21.15.119:0/3417819620":"2024-04-23T15:04:24.970842+0000","172.21.15.119:6801/2044153072":"2024-04-23T15:04:54.754937+0000","172.21.15.119:6801/3215898620":"2024-04-23T15:04:24.970842+0000","172.21.15.119:0/3856353948":"2024-04-23T15:05:49.673673+0000","172.21.15.119:6800/3215898620":"2024-04-23T15:04:24.970842+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T15:10:15.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:14 smithi186 ceph-mon[24041]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:15.590 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-22T15:10:15.591 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-22T15:10:15.591 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-22T15:10:15.592 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-22T15:10:15.592 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-22T15:10:15.592 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-22T15:10:15.593 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-22T15:10:15.593 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-22T15:10:15.974 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:15.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:15 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3768638729' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:10:16.005 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:16.005 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:16.011 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:16.012 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:16.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:15 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/3768638729' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:10:16.032 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:16.277 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:16.510 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:17.595 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:17 smithi119 ceph-mon[21835]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:17.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:17 smithi186 ceph-mon[24041]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:19.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:19 smithi119 ceph-mon[21835]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:19.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:19 smithi186 ceph-mon[24041]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:21.665 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:21 smithi119 ceph-mon[21835]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:21.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:21 smithi186 ceph-mon[24041]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:23.653 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:23 smithi119 ceph-mon[21835]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:23.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:23 smithi186 ceph-mon[24041]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:24.317 INFO:teuthology.orchestra.run.smithi119.stdout:133143986198 2024-04-22T15:10:24.317 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-22T15:10:25.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:25 smithi119 ceph-mon[21835]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:25.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:25 smithi186 ceph-mon[24041]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:27.798 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:27 smithi119 ceph-mon[21835]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:28.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:27 smithi186 ceph-mon[24041]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:29.031 INFO:teuthology.orchestra.run.smithi119.stdout:81604378658 2024-04-22T15:10:29.031 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-22T15:10:29.642 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:29 smithi119 ceph-mon[21835]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:29.980 INFO:teuthology.orchestra.run.smithi119.stdout:60129542182 2024-04-22T15:10:29.981 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-22T15:10:30.003 INFO:teuthology.orchestra.run.smithi119.stdout:167503724562 2024-04-22T15:10:30.003 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-22T15:10:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:29 smithi186 ceph-mon[24041]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:30.029 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:30.416 INFO:teuthology.orchestra.run.smithi119.stdout:201863462926 2024-04-22T15:10:30.416 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-22T15:10:30.849 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:30.849 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:30.877 INFO:teuthology.orchestra.run.smithi119.stdout:38654705709 2024-04-22T15:10:30.877 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-22T15:10:31.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:31 smithi119 ceph-mon[21835]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:31 smithi186 ceph-mon[24041]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:32.887 INFO:teuthology.orchestra.run.smithi119.stdout:236223201290 2024-04-22T15:10:32.887 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-22T15:10:33.166 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:33.739 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:33 smithi119 ceph-mon[21835]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:33.916 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:33 smithi186 ceph-mon[24041]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:34.745 INFO:teuthology.orchestra.run.smithi119.stdout:167503724564 2024-04-22T15:10:35.572 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:35 smithi119 ceph-mon[21835]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:35.572 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:35 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3789452047' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T15:10:35.594 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724562 got 167503724564 for osd.5 2024-04-22T15:10:35.595 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:36.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:35 smithi186 ceph-mon[24041]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:36.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:35 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/3789452047' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T15:10:36.033 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:36.033 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:37.652 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:37 smithi119 ceph-mon[21835]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:38.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:37 smithi186 ceph-mon[24041]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:39.640 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:39 smithi119 ceph-mon[21835]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:39.919 INFO:teuthology.orchestra.run.smithi119.stdout:38654705711 2024-04-22T15:10:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:39 smithi186 ceph-mon[24041]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:40.200 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:40.701 INFO:teuthology.orchestra.run.smithi119.stdout:111669149726 2024-04-22T15:10:40.702 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-22T15:10:40.812 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:40 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/987090753' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T15:10:41.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:40 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/987090753' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T15:10:41.522 INFO:teuthology.orchestra.run.smithi119.stdout:133143986202 2024-04-22T15:10:41.716 INFO:teuthology.orchestra.run.smithi119.stdout:60129542185 2024-04-22T15:10:41.716 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:41 smithi119 ceph-mon[21835]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:41.717 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:41 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3153797180' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T15:10:42.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:41 smithi186 ceph-mon[24041]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:42.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:41 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/3153797180' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T15:10:42.618 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705711 for osd.0 2024-04-22T15:10:42.618 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:42.877 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:42 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3375608003' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T15:10:43.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:42 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/3375608003' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T15:10:43.160 INFO:teuthology.orchestra.run.smithi119.stdout:81604378661 2024-04-22T15:10:43.394 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986198 got 133143986202 for osd.4 2024-04-22T15:10:43.394 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:43.916 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:43 smithi186 ceph-mon[24041]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:43.916 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:43 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2983655717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T15:10:43.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:43 smithi119 ceph-mon[21835]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:43.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:43 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2983655717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T15:10:44.033 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378658 got 81604378661 for osd.2 2024-04-22T15:10:44.034 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:45.572 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542182 got 60129542185 for osd.1 2024-04-22T15:10:45.572 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:45.592 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:45.953 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:45 smithi119 ceph-mon[21835]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:46.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:45 smithi186 ceph-mon[24041]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:46.883 INFO:teuthology.orchestra.run.smithi119.stdout:201863462930 2024-04-22T15:10:47.933 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:47 smithi119 ceph-mon[21835]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:47.933 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:47 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3510489378' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T15:10:47.959 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462926 got 201863462930 for osd.6 2024-04-22T15:10:47.959 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:47 smithi186 ceph-mon[24041]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:47 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/3510489378' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T15:10:48.605 INFO:teuthology.orchestra.run.smithi119.stdout:236223201293 2024-04-22T15:10:48.909 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:48 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3043355208' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T15:10:49.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:48 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/3043355208' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T15:10:49.191 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201290 got 236223201293 for osd.7 2024-04-22T15:10:49.191 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:49.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:49 smithi119 ceph-mon[21835]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:49.841 INFO:teuthology.orchestra.run.smithi119.stdout:111669149729 2024-04-22T15:10:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:49 smithi186 ceph-mon[24041]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:50.514 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149726 got 111669149729 for osd.3 2024-04-22T15:10:50.514 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:50.514 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-22T15:10:50.515 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-22T15:10:50.725 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:50.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:50 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/478838280' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T15:10:50.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:10:50.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:10:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:50 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/478838280' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T15:10:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:10:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:10:51.815 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:51 smithi119 ceph-mon[21835]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:51 smithi186 ceph-mon[24041]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:52.450 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:10:52.452 INFO:teuthology.orchestra.run.smithi119.stderr:dumped all 2024-04-22T15:10:53.097 INFO:teuthology.orchestra.run.smithi119.stdout:{"pg_ready":true,"pg_map":{"version":198,"stamp":"2024-04-22T15:10:52.130437+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48800,"kb_used_data":6304,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912416,"statfs":{"total":767960285184,"available":767910313984,"internally_reserved":0,"allocated":6455296,"data_stored":3546826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002420"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-22T15:09:58.726967+0000","last_change":"2024-04-22T15:09:58.726967+0000","last_active":"2024-04-22T15:09:58.726967+0000","last_peered":"2024-04-22T15:09:58.726967+0000","last_clean":"2024-04-22T15:09:58.726967+0000","last_became_active":"2024-04-22T15:09:58.726424+0000","last_became_peered":"2024-04-22T15:09:58.726424+0000","last_unstale":"2024-04-22T15:09:58.726967+0000","last_undegraded":"2024-04-22T15:09:58.726967+0000","last_fullsized":"2024-04-22T15:09:58.726967+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-22T15:07:56.611565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-22T15:07:56.611565+0000","last_clean_scrub_stamp":"2024-04-22T15:07:56.611565+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-23T19:49:39.572999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36599999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40100000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":19.437999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38500000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52000000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.433}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6172,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738980,"statfs":{"total":95995035648,"available":95988715520,"internally_reserved":0,"allocated":1134592,"data_stored":773573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.62}]},{"osd":1,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.57299999999999995}]},{"osd":2,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.439},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.64300000000000002}]},{"osd":3,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.51400000000000001}]},{"osd":4,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.38200000000000001}]},{"osd":5,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.60299999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45800000000000002}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:33 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.34399999999999997,"15min":0.32400000000000001},"min":{"1min":0.27600000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.186},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.35799999999999998,"15min":0.33400000000000002},"min":{"1min":0.32200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.28599999999999998}]},{"osd":2,"last update":"Mon Apr 22 15:09:59 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.38600000000000001,"15min":0.38100000000000001},"min":{"1min":0.29799999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.38300000000000001,"15min":0.379},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.52600000000000002,"5min":0.53100000000000003,"15min":0.53100000000000003},"last":0.24399999999999999}]},{"osd":3,"last update":"Mon Apr 22 15:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.437,"15min":0.439},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56200000000000006,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.46600000000000003,"15min":0.46999999999999997},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.68100000000000005,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.308}]},{"osd":4,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.58499999999999996,"15min":0.57799999999999996},"min":{"1min":0.40999999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.76000000000000001,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.58499999999999996,"15min":0.57299999999999995},"min":{"1min":0.503,"5min":0.38,"15min":0.38},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.46400000000000002}]},{"osd":5,"last update":"Mon Apr 22 15:10:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.504},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.45600000000000002}]},{"osd":6,"last update":"Mon Apr 22 15:10:31 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.43099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48299999999999998}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Apr 22 15:10:34 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.33200000000000002,"15min":0.314},"min":{"1min":0.24399999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.35499999999999998,"15min":0.32900000000000001},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":0.56699999999999995,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.40200000000000002}]},{"osd":2,"last update":"Mon Apr 22 15:10:00 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.41099999999999998,"15min":0.40300000000000002},"min":{"1min":0.30499999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.376},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.40400000000000003,"15min":0.39100000000000001},"min":{"1min":0.28899999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.29899999999999999}]},{"osd":3,"last update":"Mon Apr 22 15:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.47299999999999998,"15min":0.47499999999999998},"min":{"1min":0.32400000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.64000000000000001,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.47299999999999998,"15min":0.47399999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.63200000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Apr 22 15:10:43 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.77300000000000002,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.64500000000000002,"15min":0.64900000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.79600000000000004,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.65600000000000003}]},{"osd":5,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.64400000000000002}]},{"osd":6,"last update":"Mon Apr 22 15:10:31 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.59199999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58199999999999996}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.373,"15min":0.36899999999999999},"min":{"1min":0.23599999999999999,"5min":0.218,"15min":0.218},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.376,"15min":0.372},"min":{"1min":0.28899999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.37}]},{"osd":1,"last update":"Mon Apr 22 15:10:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.372,"15min":0.36699999999999999},"min":{"1min":0.255,"5min":0.251,"15min":0.251},"max":{"1min":0.503,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.40400000000000003,"15min":0.40000000000000002},"min":{"1min":0.26700000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.29099999999999998}]},{"osd":3,"last update":"Mon Apr 22 15:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.44900000000000001,"15min":0.45100000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.433,"5min":0.432,"15min":0.432},"min":{"1min":0.29699999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.60199999999999998,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.32800000000000001}]},{"osd":4,"last update":"Mon Apr 22 15:10:48 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.56999999999999995,"15min":0.56899999999999995},"min":{"1min":0.42299999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.755,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.57799999999999996,"15min":0.57499999999999996},"min":{"1min":0.38700000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.44400000000000001}]},{"osd":5,"last update":"Mon Apr 22 15:10:12 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.51700000000000002}]},{"osd":6,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.496},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.42099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45700000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.41099999999999998,"15min":0.41499999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.433,"15min":0.438},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.58399999999999996,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.24399999999999999}]},{"osd":1,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.42099999999999999,"15min":0.42299999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.438,"15min":0.443},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.61899999999999999,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.38300000000000001}]},{"osd":2,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.46300000000000002,"15min":0.46600000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42899999999999999,"15min":0.43099999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.30299999999999999}]},{"osd":4,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56000000000000005,"15min":0.55800000000000005},"min":{"1min":0.42199999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.38100000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.72999999999999998}]},{"osd":5,"last update":"Mon Apr 22 15:10:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.68600000000000005}]},{"osd":6,"last update":"Mon Apr 22 15:10:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.66300000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64200000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56299999999999994,"15min":0.56699999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.78900000000000003,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56999999999999995,"15min":0.57199999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73099999999999998,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.59399999999999997}]},{"osd":1,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.085,"15min":1.1719999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.78000000000000003,"5min":14.643000000000001,"15min":14.643000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.56699999999999995,"15min":0.56200000000000006},"min":{"1min":0.44600000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.75800000000000001,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.60799999999999998}]},{"osd":2,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.57199999999999995,"15min":0.57599999999999996},"min":{"1min":0.40799999999999997,"5min":0.375,"15min":0.375},"max":{"1min":0.70699999999999996,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.57699999999999996,"15min":0.57599999999999996},"min":{"1min":0.437,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80300000000000005,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.628}]},{"osd":3,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.57399999999999995,"15min":0.57099999999999995},"min":{"1min":0.433,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.77000000000000002,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.0780000000000001,"15min":1.1579999999999999},"min":{"1min":0.40699999999999997,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77100000000000002,"5min":14.618,"15min":14.618},"last":0.65600000000000003}]},{"osd":5,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Apr 22 15:10:31 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.36899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.56299999999999994}]},{"osd":1,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.65600000000000003}]},{"osd":2,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.51600000000000001}]},{"osd":3,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.59199999999999997}]},{"osd":4,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.64700000000000002}]},{"osd":6,"last update":"Mon Apr 22 15:10:31 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.628},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.63700000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-22T15:10:53.099 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-22T15:10:53.318 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:53.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:53 smithi119 ceph-mon[21835]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:53.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:53 smithi119 ceph-mon[21835]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:10:53.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:53 smithi186 ceph-mon[24041]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:53.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:53 smithi186 ceph-mon[24041]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:10:55.075 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:10:55.076 INFO:teuthology.orchestra.run.smithi119.stderr:dumped all 2024-04-22T15:10:55.715 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:55 smithi119 ceph-mon[21835]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:55.716 INFO:teuthology.orchestra.run.smithi119.stdout:{"pg_ready":true,"pg_map":{"version":199,"stamp":"2024-04-22T15:10:54.130887+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48800,"kb_used_data":6304,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912416,"statfs":{"total":767960285184,"available":767910313984,"internally_reserved":0,"allocated":6455296,"data_stored":3546826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002425"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-22T15:09:58.726967+0000","last_change":"2024-04-22T15:09:58.726967+0000","last_active":"2024-04-22T15:09:58.726967+0000","last_peered":"2024-04-22T15:09:58.726967+0000","last_clean":"2024-04-22T15:09:58.726967+0000","last_became_active":"2024-04-22T15:09:58.726424+0000","last_became_peered":"2024-04-22T15:09:58.726424+0000","last_unstale":"2024-04-22T15:09:58.726967+0000","last_undegraded":"2024-04-22T15:09:58.726967+0000","last_fullsized":"2024-04-22T15:09:58.726967+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-22T15:07:56.611565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-22T15:07:56.611565+0000","last_clean_scrub_stamp":"2024-04-22T15:07:56.611565+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-23T19:49:39.572999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36599999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40100000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":19.437999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38500000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52000000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.433}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6172,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738980,"statfs":{"total":95995035648,"available":95988715520,"internally_reserved":0,"allocated":1134592,"data_stored":773573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.62}]},{"osd":1,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.57299999999999995}]},{"osd":2,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.439},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.64300000000000002}]},{"osd":3,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.51400000000000001}]},{"osd":4,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.38200000000000001}]},{"osd":5,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.60299999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45800000000000002}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:33 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.34399999999999997,"15min":0.32400000000000001},"min":{"1min":0.27600000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.186},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.35799999999999998,"15min":0.33400000000000002},"min":{"1min":0.32200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.28599999999999998}]},{"osd":2,"last update":"Mon Apr 22 15:09:59 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.38600000000000001,"15min":0.38100000000000001},"min":{"1min":0.29799999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.38300000000000001,"15min":0.379},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.52600000000000002,"5min":0.53100000000000003,"15min":0.53100000000000003},"last":0.24399999999999999}]},{"osd":3,"last update":"Mon Apr 22 15:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.437,"15min":0.439},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56200000000000006,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.46600000000000003,"15min":0.46999999999999997},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.68100000000000005,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.308}]},{"osd":4,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.58499999999999996,"15min":0.57799999999999996},"min":{"1min":0.40999999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.76000000000000001,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.58499999999999996,"15min":0.57299999999999995},"min":{"1min":0.503,"5min":0.38,"15min":0.38},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.46400000000000002}]},{"osd":5,"last update":"Mon Apr 22 15:10:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.504},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.45600000000000002}]},{"osd":6,"last update":"Mon Apr 22 15:10:31 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.43099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48299999999999998}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Apr 22 15:10:34 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.33200000000000002,"15min":0.314},"min":{"1min":0.24399999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.249},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.35499999999999998,"15min":0.32900000000000001},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":0.56699999999999995,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.39600000000000002}]},{"osd":2,"last update":"Mon Apr 22 15:10:00 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.41099999999999998,"15min":0.40300000000000002},"min":{"1min":0.30499999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.37},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.40400000000000003,"15min":0.39100000000000001},"min":{"1min":0.28899999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.26700000000000002}]},{"osd":3,"last update":"Mon Apr 22 15:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.47299999999999998,"15min":0.47499999999999998},"min":{"1min":0.32400000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.64000000000000001,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.47299999999999998,"15min":0.47399999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.63200000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.34699999999999998}]},{"osd":4,"last update":"Mon Apr 22 15:10:43 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.77300000000000002,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.63},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.64500000000000002,"15min":0.64900000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.79600000000000004,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.95599999999999996}]},{"osd":5,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.89600000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.67200000000000004}]},{"osd":6,"last update":"Mon Apr 22 15:10:31 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.65200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63700000000000001}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.373,"15min":0.36899999999999999},"min":{"1min":0.23599999999999999,"5min":0.218,"15min":0.218},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.376,"15min":0.372},"min":{"1min":0.28899999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.37}]},{"osd":1,"last update":"Mon Apr 22 15:10:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.372,"15min":0.36699999999999999},"min":{"1min":0.255,"5min":0.251,"15min":0.251},"max":{"1min":0.503,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.40400000000000003,"15min":0.40000000000000002},"min":{"1min":0.26700000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.29099999999999998}]},{"osd":3,"last update":"Mon Apr 22 15:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.44900000000000001,"15min":0.45100000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.433,"5min":0.432,"15min":0.432},"min":{"1min":0.29699999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.60199999999999998,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.32800000000000001}]},{"osd":4,"last update":"Mon Apr 22 15:10:48 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.56999999999999995,"15min":0.56899999999999995},"min":{"1min":0.42299999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.755,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.57799999999999996,"15min":0.57499999999999996},"min":{"1min":0.38700000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.44400000000000001}]},{"osd":5,"last update":"Mon Apr 22 15:10:12 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.51700000000000002}]},{"osd":6,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.496},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.42099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45700000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.41099999999999998,"15min":0.41499999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.433,"15min":0.438},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.58399999999999996,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.26600000000000001}]},{"osd":1,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.42099999999999999,"15min":0.42299999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.377},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.438,"15min":0.443},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.61899999999999999,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.32400000000000001}]},{"osd":2,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.46300000000000002,"15min":0.46600000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42899999999999999,"15min":0.43099999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.35499999999999998}]},{"osd":4,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56000000000000005,"15min":0.55800000000000005},"min":{"1min":0.42199999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.38100000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.53100000000000003}]},{"osd":5,"last update":"Mon Apr 22 15:10:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.497},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.45200000000000001}]},{"osd":6,"last update":"Mon Apr 22 15:10:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.53800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48499999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56299999999999994,"15min":0.56699999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.78900000000000003,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56999999999999995,"15min":0.57199999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73099999999999998,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.58699999999999997}]},{"osd":1,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.085,"15min":1.1719999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.78000000000000003,"5min":14.643000000000001,"15min":14.643000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.56699999999999995,"15min":0.56200000000000006},"min":{"1min":0.44600000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.75800000000000001,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.67900000000000005}]},{"osd":2,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.57199999999999995,"15min":0.57599999999999996},"min":{"1min":0.40799999999999997,"5min":0.375,"15min":0.375},"max":{"1min":0.70699999999999996,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.57699999999999996,"15min":0.57599999999999996},"min":{"1min":0.437,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80300000000000005,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.66200000000000003}]},{"osd":3,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.57399999999999995,"15min":0.57099999999999995},"min":{"1min":0.433,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.77000000000000002,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.0780000000000001,"15min":1.1579999999999999},"min":{"1min":0.40699999999999997,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77100000000000002,"5min":14.618,"15min":14.618},"last":0.57799999999999996}]},{"osd":5,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.441},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.64000000000000001}]},{"osd":6,"last update":"Mon Apr 22 15:10:31 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.56299999999999994}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53500000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.56299999999999994}]},{"osd":1,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.65600000000000003}]},{"osd":2,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.51600000000000001}]},{"osd":3,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.59199999999999997}]},{"osd":4,"last update":"Mon Apr 22 15:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.64700000000000002}]},{"osd":6,"last update":"Mon Apr 22 15:10:31 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.628},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.63700000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-22T15:10:55.717 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-22T15:10:55.717 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-22T15:10:55.718 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-22T15:10:55.718 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-22T15:10:55.935 INFO:teuthology.orchestra.run.smithi119.stderr:Inferring config /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/mon.smithi119/config 2024-04-22T15:10:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:55 smithi186 ceph-mon[24041]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:56.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:56 smithi119 ceph-mon[21835]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:10:56.974 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:56 smithi186 ceph-mon[24041]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:10:57.737 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:10:57.737 INFO:teuthology.orchestra.run.smithi119.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-22T15:10:57.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:57 smithi119 ceph-mon[21835]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:57.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:57 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:10:57.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:57 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:10:57.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:57 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:10:57.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:57 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:10:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:57 smithi186 ceph-mon[24041]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:57 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:10:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:57 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:10:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:57 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:10:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:57 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:10:58.383 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-22T15:10:58.384 INFO:tasks.cephadm:Setup complete, yielding 2024-04-22T15:10:58.384 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T15:10:58.394 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T15:10:58.398 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch status' 2024-04-22T15:10:58.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:58 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2041678247' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T15:10:59.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:58 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2041678247' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T15:10:59.924 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:10:59 smithi119 ceph-mon[21835]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:00.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:10:59 smithi186 ceph-mon[24041]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:00.310 INFO:teuthology.orchestra.run.smithi119.stdout:Backend: cephadm 2024-04-22T15:11:00.310 INFO:teuthology.orchestra.run.smithi119.stdout:Available: Yes 2024-04-22T15:11:00.310 INFO:teuthology.orchestra.run.smithi119.stdout:Paused: No 2024-04-22T15:11:01.031 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch ps' 2024-04-22T15:11:01.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:01 smithi119 ceph-mon[21835]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:01.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:01 smithi119 ceph-mon[21835]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:01 smithi186 ceph-mon[24041]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:01 smithi186 ceph-mon[24041]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:02.844 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:11:02.844 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (4m) 2m ago 5m 15.5M - ba2b418f427c 0e23a591f269 2024-04-22T15:11:02.844 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e 37eb54a66662 2024-04-22T15:11:02.844 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (5m) 69s ago 5m 6405k - 17.2.0 e1d6a67b021e ee37991f5d20 2024-04-22T15:11:02.844 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (4m) 2m ago 5m 45.1M - 8.3.5 dad864ee21e9 a8f43d2ae83a 2024-04-22T15:11:02.844 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:9283 running (6m) 2m ago 6m 447M - 17.2.0 e1d6a67b021e bac90840f90b 2024-04-22T15:11:02.844 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283 running (5m) 69s ago 5m 411M - 17.2.0 e1d6a67b021e b36d3b07de5e 2024-04-22T15:11:02.844 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (6m) 2m ago 7m 41.2M 2048M 17.2.0 e1d6a67b021e e31b6591d879 2024-04-22T15:11:02.844 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (4m) 69s ago 4m 42.0M 2048M 17.2.0 e1d6a67b021e a804e52c2c5e 2024-04-22T15:11:02.845 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (5m) 2m ago 5m 16.5M - 1dbe0e931976 2499dd8265c7 2024-04-22T15:11:02.845 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (4m) 69s ago 4m 17.1M - 1dbe0e931976 5a18b815fb1f 2024-04-22T15:11:02.845 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (4m) 2m ago 4m 32.9M 3216M 17.2.0 e1d6a67b021e 9999e9684eba 2024-04-22T15:11:02.845 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (3m) 2m ago 3m 37.0M 3216M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:11:02.845 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (3m) 2m ago 3m 31.4M 3216M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:11:02.845 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (2m) 2m ago 2m 10.3M 3216M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:11:02.845 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (2m) 69s ago 2m 33.6M 3984M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:11:02.845 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (2m) 69s ago 2m 33.3M 3984M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:11:02.845 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (99s) 69s ago 98s 30.6M 3984M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:11:02.845 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (74s) 69s ago 73s 10.4M 3984M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:11:02.845 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (4m) 2m ago 4m 53.9M - 514e6a882f6e b8d42c7e1876 2024-04-22T15:11:03.538 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch ls' 2024-04-22T15:11:03.848 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:03 smithi119 ceph-mon[21835]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:03.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:03 smithi186 ceph-mon[24041]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:04.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:04 smithi119 ceph-mon[21835]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:05.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:04 smithi186 ceph-mon[24041]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:05.675 INFO:teuthology.orchestra.run.smithi119.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-22T15:11:05.675 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-04-22T15:11:05.675 INFO:teuthology.orchestra.run.smithi119.stdout:crash 2/2 2m ago 6m * 2024-04-22T15:11:05.675 INFO:teuthology.orchestra.run.smithi119.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-04-22T15:11:05.675 INFO:teuthology.orchestra.run.smithi119.stdout:mgr 2/2 2m ago 6m count:2 2024-04-22T15:11:05.675 INFO:teuthology.orchestra.run.smithi119.stdout:mon 2/2 2m ago 5m smithi119:172.21.15.119=smithi119;smithi186:172.21.15.186=smithi186;count:2 2024-04-22T15:11:05.675 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-04-22T15:11:05.675 INFO:teuthology.orchestra.run.smithi119.stdout:osd 8 2m ago - 2024-04-22T15:11:05.675 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-04-22T15:11:05.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:05 smithi186 ceph-mon[24041]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:05.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:05 smithi119 ceph-mon[21835]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:06.317 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch host ls' 2024-04-22T15:11:06.946 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:06 smithi119 ceph-mon[21835]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:07.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:06 smithi186 ceph-mon[24041]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:07.885 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:07 smithi119 ceph-mon[21835]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:07 smithi186 ceph-mon[24041]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:08.165 INFO:teuthology.orchestra.run.smithi119.stdout:HOST ADDR LABELS STATUS 2024-04-22T15:11:08.165 INFO:teuthology.orchestra.run.smithi119.stdout:smithi119 172.21.15.119 2024-04-22T15:11:08.165 INFO:teuthology.orchestra.run.smithi119.stdout:smithi186 172.21.15.186 2024-04-22T15:11:08.165 INFO:teuthology.orchestra.run.smithi119.stdout:2 hosts in cluster 2024-04-22T15:11:08.848 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch device ls' 2024-04-22T15:11:09.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:09 smithi119 ceph-mon[21835]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:09.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:09 smithi119 ceph-mon[21835]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:10.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:09 smithi186 ceph-mon[24041]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:10.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:09 smithi186 ceph-mon[24041]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:10.725 INFO:teuthology.orchestra.run.smithi119.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-04-22T15:11:10.725 INFO:teuthology.orchestra.run.smithi119.stdout:smithi119 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400Y1400BGN 400G LVM detected, locked 2024-04-22T15:11:10.725 INFO:teuthology.orchestra.run.smithi119.stdout:smithi186 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT623300CB400BGN 400G LVM detected, locked 2024-04-22T15:11:11.404 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-22T15:11:11.416 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T15:11:11.416 DEBUG:teuthology.orchestra.run.smithi119:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-22T15:11:11.442 INFO:teuthology.orchestra.run.smithi119.stderr:+ systemctl stop nfs-server 2024-04-22T15:11:11.449 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi186.front.sepia.ceph.com 2024-04-22T15:11:11.450 DEBUG:teuthology.orchestra.run.smithi186:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-22T15:11:11.480 INFO:teuthology.orchestra.run.smithi186.stderr:+ systemctl stop nfs-server 2024-04-22T15:11:11.489 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T15:11:11.498 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T15:11:11.498 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-04-22T15:11:11.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:11 smithi186 ceph-mon[24041]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:11.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:11 smithi186 ceph-mon[24041]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:11.800 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:11 smithi119 ceph-mon[21835]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:11.801 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:11 smithi119 ceph-mon[21835]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:13.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:13 smithi119 ceph-mon[21835]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:13 smithi186 ceph-mon[24041]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:14.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:14 smithi119 ceph-mon[21835]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:14.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:14 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-22T15:11:14.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:14 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-22T15:11:14.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:14 smithi119 ceph-mon[21835]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T15:11:14.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:14 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-22T15:11:14.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:14 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119[21831]: 2024-04-22T15:11:14.746+0000 7f4bf94e0700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T15:11:15.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:14 smithi186 ceph-mon[24041]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:15.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:14 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-22T15:11:15.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:14 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-22T15:11:15.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:14 smithi186 ceph-mon[24041]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T15:11:15.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:14 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-22T15:11:15.499 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-04-22T15:11:15.509 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-04-22T15:11:15.509 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-22T15:11:16.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: pgmap v210: 33 pgs: 2 creating+peering, 1 active+clean, 30 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:16.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-22T15:11:16.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T15:11:16.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: fsmap foofs:0 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi119.ncbqaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi119.ncbqaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-22T15:11:16.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:15 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:16.062 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: pgmap v210: 33 pgs: 2 creating+peering, 1 active+clean, 30 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:16.062 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-22T15:11:16.062 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T15:11:16.062 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-22T15:11:16.062 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T15:11:16.062 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-22T15:11:16.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T15:11:16.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-22T15:11:16.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: fsmap foofs:0 2024-04-22T15:11:16.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:16.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:16.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:16.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:16.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:16.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:16.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi119.ncbqaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T15:11:16.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi119.ncbqaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-22T15:11:16.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:15 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:17.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:16 smithi186 ceph-mon[24041]: Saving service mds.foofs spec with placement count:2 2024-04-22T15:11:17.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:16 smithi186 ceph-mon[24041]: Deploying daemon mds.foofs.smithi119.ncbqaf on smithi119 2024-04-22T15:11:17.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:16 smithi186 ceph-mon[24041]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T15:11:17.151 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:16 smithi119 ceph-mon[21835]: Saving service mds.foofs spec with placement count:2 2024-04-22T15:11:17.151 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:16 smithi119 ceph-mon[21835]: Deploying daemon mds.foofs.smithi119.ncbqaf on smithi119 2024-04-22T15:11:17.151 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:16 smithi119 ceph-mon[21835]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T15:11:17.889 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:17 smithi119 ceph-mon[21835]: pgmap v214: 65 pgs: 12 active+clean, 13 creating+peering, 40 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:17.889 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:17 smithi119 ceph-mon[21835]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T15:11:18.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:17 smithi186 ceph-mon[24041]: pgmap v214: 65 pgs: 12 active+clean, 13 creating+peering, 40 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:18.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:17 smithi186 ceph-mon[24041]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T15:11:18.525 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:11:18.525 INFO:teuthology.orchestra.run.smithi119.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-22T15:04:44.298348Z", "last_refresh": "2024-04-22T15:08:22.812410Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-22T15:04:37.826787Z", "last_refresh": "2024-04-22T15:08:22.812491Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-22T15:04:41.840104Z", "last_refresh": "2024-04-22T15:08:22.812549Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-22T15:11:14.768692Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-22T15:11:14.762937Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-22T15:04:35.847067Z", "last_refresh": "2024-04-22T15:08:22.812338Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi119:172.21.15.119=smithi119", "smithi186:172.21.15.186=smithi186"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-22T15:05:26.289192Z", "last_refresh": "2024-04-22T15:08:22.812183Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-22T15:04:43.132534Z", "last_refresh": "2024-04-22T15:08:22.812607Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-22T15:08:22.812725Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-22T15:04:39.724425Z", "last_refresh": "2024-04-22T15:08:22.812667Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-22T15:11:19.283 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-22T15:11:19.976 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:19 smithi186 ceph-mon[24041]: pgmap v216: 65 pgs: 40 active+clean, 13 creating+peering, 12 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:19.976 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:19 smithi186 ceph-mon[24041]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:11:19.976 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:19 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:19.976 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:19 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi186.dglszf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T15:11:19.976 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:19 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi186.dglszf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-22T15:11:19.977 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:19 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:20.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:19 smithi119 ceph-mon[21835]: pgmap v216: 65 pgs: 40 active+clean, 13 creating+peering, 12 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:20.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:19 smithi119 ceph-mon[21835]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:11:20.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:19 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:20.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:19 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi186.dglszf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T15:11:20.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:19 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi186.dglszf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-22T15:11:20.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:19 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:20.285 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-22T15:11:20.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:20 smithi119 ceph-mon[21835]: Deploying daemon mds.foofs.smithi186.dglszf on smithi186 2024-04-22T15:11:20.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:20 smithi119 ceph-mon[21835]: mds.? [v2:172.21.15.119:6834/3732854443,v1:172.21.15.119:6835/3732854443] up:boot 2024-04-22T15:11:20.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:20 smithi119 ceph-mon[21835]: daemon mds.foofs.smithi119.ncbqaf assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-22T15:11:20.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:20 smithi119 ceph-mon[21835]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-22T15:11:20.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:20 smithi119 ceph-mon[21835]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-22T15:11:20.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:20 smithi119 ceph-mon[21835]: Cluster is now healthy 2024-04-22T15:11:20.897 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:20 smithi119 ceph-mon[21835]: fsmap foofs:0 1 up:standby 2024-04-22T15:11:20.897 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:20 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:11:20.897 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:20 smithi119 ceph-mon[21835]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:creating} 2024-04-22T15:11:20.897 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:20 smithi119 ceph-mon[21835]: daemon mds.foofs.smithi119.ncbqaf is now active in filesystem foofs as rank 0 2024-04-22T15:11:21.020 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:20 smithi186 ceph-mon[24041]: Deploying daemon mds.foofs.smithi186.dglszf on smithi186 2024-04-22T15:11:21.020 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:20 smithi186 ceph-mon[24041]: mds.? [v2:172.21.15.119:6834/3732854443,v1:172.21.15.119:6835/3732854443] up:boot 2024-04-22T15:11:21.020 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:20 smithi186 ceph-mon[24041]: daemon mds.foofs.smithi119.ncbqaf assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-22T15:11:21.020 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:20 smithi186 ceph-mon[24041]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-22T15:11:21.020 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:20 smithi186 ceph-mon[24041]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-22T15:11:21.020 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:20 smithi186 ceph-mon[24041]: Cluster is now healthy 2024-04-22T15:11:21.020 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:20 smithi186 ceph-mon[24041]: fsmap foofs:0 1 up:standby 2024-04-22T15:11:21.020 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:20 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:11:21.020 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:20 smithi186 ceph-mon[24041]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:creating} 2024-04-22T15:11:21.020 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:20 smithi186 ceph-mon[24041]: daemon mds.foofs.smithi119.ncbqaf is now active in filesystem foofs as rank 0 2024-04-22T15:11:21.870 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:21 smithi119 ceph-mon[21835]: pgmap v217: 65 pgs: 60 active+clean, 5 creating+peering; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:21.870 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:21 smithi119 ceph-mon[21835]: mds.? [v2:172.21.15.119:6834/3732854443,v1:172.21.15.119:6835/3732854443] up:active 2024-04-22T15:11:21.870 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:21 smithi119 ceph-mon[21835]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 2024-04-22T15:11:22.009 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:21 smithi186 ceph-mon[24041]: pgmap v217: 65 pgs: 60 active+clean, 5 creating+peering; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:22.010 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:21 smithi186 ceph-mon[24041]: mds.? [v2:172.21.15.119:6834/3732854443,v1:172.21.15.119:6835/3732854443] up:active 2024-04-22T15:11:22.010 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:21 smithi186 ceph-mon[24041]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 2024-04-22T15:11:22.178 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:11:22.178 INFO:teuthology.orchestra.run.smithi119.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-22T15:04:44.298348Z", "last_refresh": "2024-04-22T15:08:22.812410Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-22T15:04:37.826787Z", "last_refresh": "2024-04-22T15:08:22.812491Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-22T15:04:41.840104Z", "last_refresh": "2024-04-22T15:08:22.812549Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-22T15:11:14.768692Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-22T15:11:14.762937Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-22T15:04:35.847067Z", "last_refresh": "2024-04-22T15:08:22.812338Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi119:172.21.15.119=smithi119", "smithi186:172.21.15.186=smithi186"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-22T15:05:26.289192Z", "last_refresh": "2024-04-22T15:08:22.812183Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-22T15:04:43.132534Z", "last_refresh": "2024-04-22T15:08:22.812607Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-22T15:08:22.812725Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-22T15:04:39.724425Z", "last_refresh": "2024-04-22T15:08:22.812667Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-22T15:11:22.704 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-22T15:11:23.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:22 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:23.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:22 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:23.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:22 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:23.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:22 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:23.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:22 smithi119 ceph-mon[21835]: pgmap v218: 65 pgs: 65 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 693 B/s wr, 3 op/s 2024-04-22T15:11:23.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:22 smithi119 ceph-mon[21835]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:11:23.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:22 smithi119 ceph-mon[21835]: mds.? [v2:172.21.15.186:6832/2825961187,v1:172.21.15.186:6833/2825961187] up:boot 2024-04-22T15:11:23.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:22 smithi119 ceph-mon[21835]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 1 up:standby 2024-04-22T15:11:23.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:22 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:11:23.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:22 smithi119 ceph-mon[21835]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 1 up:standby 2024-04-22T15:11:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:22 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:22 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:22 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:22 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:22 smithi186 ceph-mon[24041]: pgmap v218: 65 pgs: 65 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 693 B/s wr, 3 op/s 2024-04-22T15:11:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:22 smithi186 ceph-mon[24041]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:11:23.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:22 smithi186 ceph-mon[24041]: mds.? [v2:172.21.15.186:6832/2825961187,v1:172.21.15.186:6833/2825961187] up:boot 2024-04-22T15:11:23.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:22 smithi186 ceph-mon[24041]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 1 up:standby 2024-04-22T15:11:23.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:22 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:11:23.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:22 smithi186 ceph-mon[24041]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 1 up:standby 2024-04-22T15:11:23.706 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-22T15:11:25.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:24 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:25.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:24 smithi119 ceph-mon[21835]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 4 op/s 2024-04-22T15:11:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:24 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:24 smithi186 ceph-mon[24041]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 4 op/s 2024-04-22T15:11:26.437 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:26 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:26.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:26 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:26.737 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:11:26.737 INFO:teuthology.orchestra.run.smithi119.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-22T15:04:44.298348Z", "last_refresh": "2024-04-22T15:08:22.812410Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-22T15:04:37.826787Z", "last_refresh": "2024-04-22T15:08:22.812491Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-22T15:04:41.840104Z", "last_refresh": "2024-04-22T15:08:22.812549Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-22T15:11:14.768692Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-22T15:11:14.762937Z", "last_refresh": "2024-04-22T15:11:23.873909Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-22T15:04:35.847067Z", "last_refresh": "2024-04-22T15:08:22.812338Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi119:172.21.15.119=smithi119", "smithi186:172.21.15.186=smithi186"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-22T15:05:26.289192Z", "last_refresh": "2024-04-22T15:08:22.812183Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-22T15:04:43.132534Z", "last_refresh": "2024-04-22T15:08:22.812607Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-22T15:08:22.812725Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-22T15:04:39.724425Z", "last_refresh": "2024-04-22T15:08:22.812667Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-22T15:11:27.778 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:27 smithi119 ceph-mon[21835]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-22T15:11:27.778 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:27 smithi119 ceph-mon[21835]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:11:27.802 INFO:tasks.cephadm:mds.foofs has 1/2 2024-04-22T15:11:28.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:27 smithi186 ceph-mon[24041]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-22T15:11:28.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:27 smithi186 ceph-mon[24041]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:11:28.803 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-22T15:11:29.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:29 smithi119 ceph-mon[21835]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-22T15:11:29.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:29 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:29.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:29 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:29.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:29 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:29.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:29 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:29.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:29 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:29.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:29 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:29 smithi186 ceph-mon[24041]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-22T15:11:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:29 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:29 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:29 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:29 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:30.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:29 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:30.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:29 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:30.812 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:11:30.812 INFO:teuthology.orchestra.run.smithi119.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-22T15:04:44.298348Z", "last_refresh": "2024-04-22T15:11:28.669713Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-22T15:04:37.826787Z", "last_refresh": "2024-04-22T15:11:23.873291Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-22T15:04:41.840104Z", "last_refresh": "2024-04-22T15:11:28.669835Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-22T15:11:14.768692Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-22T15:11:14.762937Z", "last_refresh": "2024-04-22T15:11:23.873909Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-22T15:04:35.847067Z", "last_refresh": "2024-04-22T15:11:23.873445Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi119:172.21.15.119=smithi119", "smithi186:172.21.15.186=smithi186"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-22T15:05:26.289192Z", "last_refresh": "2024-04-22T15:11:23.873523Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-22T15:04:43.132534Z", "last_refresh": "2024-04-22T15:11:23.873604Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-22T15:11:23.873666Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-22T15:04:39.724425Z", "last_refresh": "2024-04-22T15:11:28.669952Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-22T15:11:31.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:31 smithi119 ceph-mon[21835]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-22T15:11:31.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:31 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:31.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:31 smithi186 ceph-mon[24041]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-22T15:11:31.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:31 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:31.538 INFO:tasks.cephadm:mds.foofs has 2/2 2024-04-22T15:11:31.539 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T15:11:31.551 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T15:11:31.551 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-04-22T15:11:32.720 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:32 smithi119 ceph-mon[21835]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:11:33.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:32 smithi186 ceph-mon[24041]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:11:33.916 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:33 smithi186 ceph-mon[24041]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-22T15:11:33.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:33 smithi119 ceph-mon[21835]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-22T15:11:34.740 INFO:teuthology.orchestra.run.smithi119.stdout:NFS Cluster Created Successfully 2024-04-22T15:11:34.936 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:34 smithi119 ceph-mon[21835]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:34.936 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-22T15:11:34.936 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-22T15:11:34.936 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:34 smithi119 ceph-mon[21835]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T15:11:34.936 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:34 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-22T15:11:35.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:34 smithi186 ceph-mon[24041]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:35.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-22T15:11:35.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-22T15:11:35.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:34 smithi186 ceph-mon[24041]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T15:11:35.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:34 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-22T15:11:35.376 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-04-22T15:11:35.905 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: pgmap v225: 97 pgs: 5 creating+peering, 27 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-22T15:11:35.905 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-22T15:11:35.905 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T15:11:35.905 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: Saving service nfs.foo spec with placement count:2 2024-04-22T15:11:35.905 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:35.905 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:35.905 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:35.905 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:35.905 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:35.905 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:35.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi119.zcjqol", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T15:11:35.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi119.zcjqol", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-22T15:11:35.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T15:11:35.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T15:11:35.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:35.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T15:11:35.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T15:11:35.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi119.zcjqol-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:11:35.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi119.zcjqol-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T15:11:35.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:35 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:36.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: pgmap v225: 97 pgs: 5 creating+peering, 27 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-22T15:11:36.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-22T15:11:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T15:11:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: Saving service nfs.foo spec with placement count:2 2024-04-22T15:11:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi119.zcjqol", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T15:11:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi119.zcjqol", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-22T15:11:36.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T15:11:36.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T15:11:36.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:36.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T15:11:36.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T15:11:36.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi119.zcjqol-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:11:36.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi119.zcjqol-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T15:11:36.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:35 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:36.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:36 smithi119 ceph-mon[21835]: Creating key for client.nfs.foo.0.0.smithi119.zcjqol 2024-04-22T15:11:36.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:36 smithi119 ceph-mon[21835]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-22T15:11:36.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:36 smithi119 ceph-mon[21835]: Rados config object exists: conf-nfs.foo 2024-04-22T15:11:36.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:36 smithi119 ceph-mon[21835]: Creating key for client.nfs.foo.0.0.smithi119.zcjqol-rgw 2024-04-22T15:11:36.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:36 smithi119 ceph-mon[21835]: Deploying daemon nfs.foo.0.0.smithi119.zcjqol on smithi119 2024-04-22T15:11:36.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:36 smithi119 ceph-mon[21835]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T15:11:37.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:36 smithi186 ceph-mon[24041]: Creating key for client.nfs.foo.0.0.smithi119.zcjqol 2024-04-22T15:11:37.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:36 smithi186 ceph-mon[24041]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-22T15:11:37.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:36 smithi186 ceph-mon[24041]: Rados config object exists: conf-nfs.foo 2024-04-22T15:11:37.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:36 smithi186 ceph-mon[24041]: Creating key for client.nfs.foo.0.0.smithi119.zcjqol-rgw 2024-04-22T15:11:37.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:36 smithi186 ceph-mon[24041]: Deploying daemon nfs.foo.0.0.smithi119.zcjqol on smithi119 2024-04-22T15:11:37.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:36 smithi186 ceph-mon[24041]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T15:11:37.974 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:37 smithi119 ceph-mon[21835]: pgmap v228: 97 pgs: 5 creating+peering, 19 unknown, 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:37.974 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:37 smithi119 ceph-mon[21835]: mgrmap e20: smithi119.jjbxxd(active, since 5m), standbys: smithi186.vtjgwg 2024-04-22T15:11:38.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:37 smithi186 ceph-mon[24041]: pgmap v228: 97 pgs: 5 creating+peering, 19 unknown, 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:38.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:37 smithi186 ceph-mon[24041]: mgrmap e20: smithi119.jjbxxd(active, since 5m), standbys: smithi186.vtjgwg 2024-04-22T15:11:38.630 INFO:teuthology.orchestra.run.smithi119.stderr:Invalid command: Unexpected argument '--clusterid' 2024-04-22T15:11:38.630 INFO:teuthology.orchestra.run.smithi119.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-04-22T15:11:38.631 INFO:teuthology.orchestra.run.smithi119.stderr:Error EINVAL: invalid command 2024-04-22T15:11:38.994 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:11:38.994 INFO:teuthology.orchestra.run.smithi119.stdout: "bind": "/fake", 2024-04-22T15:11:38.994 INFO:teuthology.orchestra.run.smithi119.stdout: "fs": "foofs", 2024-04-22T15:11:38.994 INFO:teuthology.orchestra.run.smithi119.stdout: "path": "/", 2024-04-22T15:11:38.995 INFO:teuthology.orchestra.run.smithi119.stdout: "cluster": "foo", 2024-04-22T15:11:38.995 INFO:teuthology.orchestra.run.smithi119.stdout: "mode": "RW" 2024-04-22T15:11:38.995 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:11:39.775 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-04-22T15:11:39.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:39 smithi119 ceph-mon[21835]: pgmap v229: 97 pgs: 5 creating+peering, 92 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-22T15:11:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:39 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:39 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi186.mklmhy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T15:11:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:39 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi186.mklmhy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-22T15:11:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:39 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-22T15:11:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:39 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T15:11:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:39 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-22T15:11:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:39 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T15:11:39.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:39 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:39 smithi186 ceph-mon[24041]: pgmap v229: 97 pgs: 5 creating+peering, 92 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-22T15:11:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:39 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:39 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi186.mklmhy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T15:11:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:39 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi186.mklmhy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-22T15:11:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:39 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-22T15:11:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:39 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T15:11:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:39 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-22T15:11:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:39 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T15:11:40.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:39 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:40.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:40 smithi119 ceph-mon[21835]: from='client.14530 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:40.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:40 smithi119 ceph-mon[21835]: Creating key for client.nfs.foo.1.0.smithi186.mklmhy 2024-04-22T15:11:40.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:40 smithi119 ceph-mon[21835]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-22T15:11:41.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:40 smithi186 ceph-mon[24041]: from='client.14530 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:41.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:40 smithi186 ceph-mon[24041]: Creating key for client.nfs.foo.1.0.smithi186.mklmhy 2024-04-22T15:11:41.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:40 smithi186 ceph-mon[24041]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-22T15:11:41.874 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:41 smithi119 ceph-mon[21835]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 158 B/s rd, 317 B/s wr, 0 op/s 2024-04-22T15:11:41.978 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:41 smithi186 ceph-mon[24041]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 158 B/s rd, 317 B/s wr, 0 op/s 2024-04-22T15:11:42.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:42 smithi119 ceph-mon[21835]: from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:42.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:42 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T15:11:42.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:42 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T15:11:42.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:42 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi186.mklmhy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:11:42.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:42 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi186.mklmhy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T15:11:42.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:42 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:43.012 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:42 smithi186 ceph-mon[24041]: from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:43.012 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:42 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T15:11:43.012 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:42 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T15:11:43.013 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:42 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi186.mklmhy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:11:43.013 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:42 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi186.mklmhy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T15:11:43.013 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:42 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:43.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:43 smithi186 ceph-mon[24041]: Rados config object exists: conf-nfs.foo 2024-04-22T15:11:43.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:43 smithi186 ceph-mon[24041]: Creating key for client.nfs.foo.1.0.smithi186.mklmhy-rgw 2024-04-22T15:11:43.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:43 smithi186 ceph-mon[24041]: Deploying daemon nfs.foo.1.0.smithi186.mklmhy on smithi186 2024-04-22T15:11:43.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:43 smithi186 ceph-mon[24041]: pgmap v231: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 383 B/s wr, 0 op/s 2024-04-22T15:11:43.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:43 smithi119 ceph-mon[21835]: Rados config object exists: conf-nfs.foo 2024-04-22T15:11:43.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:43 smithi119 ceph-mon[21835]: Creating key for client.nfs.foo.1.0.smithi186.mklmhy-rgw 2024-04-22T15:11:43.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:43 smithi119 ceph-mon[21835]: Deploying daemon nfs.foo.1.0.smithi186.mklmhy on smithi186 2024-04-22T15:11:43.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:43 smithi119 ceph-mon[21835]: pgmap v231: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 383 B/s wr, 0 op/s 2024-04-22T15:11:44.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:44 smithi119 ceph-mon[21835]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:44.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:44 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:44.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:44 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:44.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:44 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:44.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:44 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:44.992 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:44 smithi186 ceph-mon[24041]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:44.992 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:44 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:44.992 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:44 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:44.993 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:44 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:44.993 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:44 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:45.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:45 smithi186 ceph-mon[24041]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-22T15:11:45.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:45 smithi186 ceph-mon[24041]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:45.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:45 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:45.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:45 smithi119 ceph-mon[21835]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-22T15:11:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:45 smithi119 ceph-mon[21835]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:45.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:45 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:47.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:46 smithi186 ceph-mon[24041]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:47.029 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:46 smithi119 ceph-mon[21835]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:47.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:47 smithi119 ceph-mon[21835]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-04-22T15:11:47.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:47 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:47 smithi186 ceph-mon[24041]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-04-22T15:11:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:47 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:48.970 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:48 smithi119 ceph-mon[21835]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:49.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:48 smithi186 ceph-mon[24041]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:49.627 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-04-22T15:11:49.951 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:49 smithi119 ceph-mon[21835]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-04-22T15:11:49.951 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:49 smithi119 ceph-mon[21835]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:49.951 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:49 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:49.951 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:49 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:49.951 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:49 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:49.951 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:49 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:49.951 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:49 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:49.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:49 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:49.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:49 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:49.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:49 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:49 smithi186 ceph-mon[24041]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-04-22T15:11:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:49 smithi186 ceph-mon[24041]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:49 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:49 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:49 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:50.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:49 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:50.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:49 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:50.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:49 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:50.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:49 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:50.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:49 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:11:50.291 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-22T15:11:50.301 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T15:11:50.302 DEBUG:teuthology.orchestra.run.smithi119:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-04-22T15:11:50.325 INFO:teuthology.orchestra.run.smithi119.stderr:+ mkdir /mnt/foo 2024-04-22T15:11:50.337 DEBUG:teuthology.orchestra.run.smithi119:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-04-22T15:11:50.399 INFO:teuthology.orchestra.run.smithi119.stderr:++ hostname 2024-04-22T15:11:50.399 INFO:teuthology.orchestra.run.smithi119.stderr:+ mount -t nfs smithi119:/fake /mnt/foo -o sync 2024-04-22T15:11:50.774 DEBUG:teuthology.orchestra.run.smithi119:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-04-22T15:11:50.835 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:50 smithi119 ceph-mon[21835]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:50.835 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-04-22T15:11:50.835 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-22T15:11:50.835 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-04-22T15:11:50.835 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-04-22T15:11:50.835 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:11:50.835 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:11:50.837 INFO:teuthology.orchestra.run.smithi119.stderr:+ echo test 2024-04-22T15:11:50.881 DEBUG:teuthology.orchestra.run.smithi119:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-04-22T15:11:50.943 INFO:teuthology.orchestra.run.smithi119.stderr:+ sync 2024-04-22T15:11:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:50 smithi186 ceph-mon[24041]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-04-22T15:11:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-22T15:11:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-04-22T15:11:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-04-22T15:11:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:11:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:11:51.170 INFO:teuthology.run_tasks:Running task parallel... 2024-04-22T15:11:51.201 INFO:teuthology.task.parallel:starting parallel... 2024-04-22T15:11:51.202 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-22T15:11:51.203 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-22T15:11:51.203 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T15:11:51.203 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-22T15:11:51.204 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-22T15:11:51.204 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-04-22T15:11:51.205 INFO:teuthology.task.exec:Executing custom commands... 2024-04-22T15:11:51.205 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T15:11:51.205 DEBUG:teuthology.orchestra.run.smithi119:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-04-22T15:11:51.320 INFO:teuthology.orchestra.run.smithi119.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-22T15:11:51.320 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:11:51.320 INFO:teuthology.orchestra.run.smithi119.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-04-22T15:11:51.321 INFO:teuthology.orchestra.run.smithi119.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-22T15:11:51.421 INFO:teuthology.orchestra.run.smithi119.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-22T15:11:51.421 INFO:teuthology.orchestra.run.smithi119.stdout:releasing clients 2024-04-22T15:11:51.976 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:51 smithi119 ceph-mon[21835]: pgmap v235: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-04-22T15:11:51.977 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:51 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-04-22T15:11:51.977 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:51 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-04-22T15:11:51.977 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:51 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2024-04-22T15:11:51.977 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:51 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-04-22T15:11:51.977 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:51 smithi119 ceph-mon[21835]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T15:11:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:51 smithi186 ceph-mon[24041]: pgmap v235: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-04-22T15:11:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:51 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-04-22T15:11:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:51 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-04-22T15:11:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:51 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2024-04-22T15:11:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:51 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-04-22T15:11:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:51 smithi186 ceph-mon[24041]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T15:11:52.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 115 26.08 MB/sec warmup 1 sec latency 31.097 ms 2024-04-22T15:11:52.779 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:52 smithi119 ceph-mon[21835]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T15:11:52.779 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:52 smithi119 ceph-mon[21835]: pgmap v238: 97 pgs: 1 peering, 96 active+clean; 2.8 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 291 KiB/s wr, 10 op/s 2024-04-22T15:11:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:52 smithi186 ceph-mon[24041]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T15:11:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:52 smithi186 ceph-mon[24041]: pgmap v238: 97 pgs: 1 peering, 96 active+clean; 2.8 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 291 KiB/s wr, 10 op/s 2024-04-22T15:11:53.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 241 22.99 MB/sec warmup 2 sec latency 31.468 ms 2024-04-22T15:11:54.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 356 21.20 MB/sec warmup 3 sec latency 53.637 ms 2024-04-22T15:11:54.685 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-22T15:11:55.345 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:55 smithi119 ceph-mon[21835]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 19 MiB data, 53 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.3 MiB/s wr, 45 op/s; 2 B/s, 0 objects/s recovering 2024-04-22T15:11:55.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 468 20.47 MB/sec warmup 4 sec latency 39.888 ms 2024-04-22T15:11:55.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:55 smithi186 ceph-mon[24041]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 19 MiB data, 53 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.3 MiB/s wr, 45 op/s; 2 B/s, 0 objects/s recovering 2024-04-22T15:11:56.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 576 19.65 MB/sec warmup 5 sec latency 38.523 ms 2024-04-22T15:11:57.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 891 18.06 MB/sec warmup 6 sec latency 23.957 ms 2024-04-22T15:11:57.785 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:57 smithi119 ceph-mon[21835]: pgmap v240: 97 pgs: 1 peering, 96 active+clean; 49 MiB data, 80 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s wr, 296 op/s; 2 B/s, 0 objects/s recovering 2024-04-22T15:11:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:57 smithi186 ceph-mon[24041]: pgmap v240: 97 pgs: 1 peering, 96 active+clean; 49 MiB data, 80 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s wr, 296 op/s; 2 B/s, 0 objects/s recovering 2024-04-22T15:11:58.210 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-22T15:11:58.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 1181 16.45 MB/sec warmup 7 sec latency 85.316 ms 2024-04-22T15:11:59.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 2065 16.64 MB/sec warmup 8 sec latency 23.091 ms 2024-04-22T15:11:59.552 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:11:59 smithi119 ceph-mon[21835]: pgmap v241: 97 pgs: 97 active+clean; 85 MiB data, 142 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 11 MiB/s wr, 508 op/s; 2 B/s, 0 objects/s recovering 2024-04-22T15:12:00.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:11:59 smithi186 ceph-mon[24041]: pgmap v241: 97 pgs: 97 active+clean; 85 MiB data, 142 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 11 MiB/s wr, 508 op/s; 2 B/s, 0 objects/s recovering 2024-04-22T15:12:00.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 2841 18.04 MB/sec warmup 9 sec latency 21.082 ms 2024-04-22T15:12:01.422 INFO:teuthology.orchestra.run.smithi119.stdout: 5 3519 18.66 MB/sec warmup 10 sec latency 38.265 ms 2024-04-22T15:12:01.621 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:01 smithi119 ceph-mon[21835]: pgmap v242: 97 pgs: 97 active+clean; 109 MiB data, 173 MiB used, 715 GiB / 715 GiB avail; 218 B/s rd, 12 MiB/s wr, 586 op/s; 2 B/s, 0 objects/s recovering 2024-04-22T15:12:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:01 smithi186 ceph-mon[24041]: pgmap v242: 97 pgs: 97 active+clean; 109 MiB data, 173 MiB used, 715 GiB / 715 GiB avail; 218 B/s rd, 12 MiB/s wr, 586 op/s; 2 B/s, 0 objects/s recovering 2024-04-22T15:12:02.423 INFO:teuthology.orchestra.run.smithi119.stdout: 5 3890 18.13 MB/sec warmup 11 sec latency 91.527 ms 2024-04-22T15:12:03.366 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph mgr module enable nfs --force' 2024-04-22T15:12:03.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 4493 17.03 MB/sec warmup 12 sec latency 32.985 ms 2024-04-22T15:12:03.642 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:03 smithi119 ceph-mon[21835]: pgmap v243: 97 pgs: 97 active+clean; 114 MiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 12 MiB/s wr, 568 op/s; 2 B/s, 0 objects/s recovering 2024-04-22T15:12:03.915 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:03 smithi186 ceph-mon[24041]: pgmap v243: 97 pgs: 97 active+clean; 114 MiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 12 MiB/s wr, 568 op/s; 2 B/s, 0 objects/s recovering 2024-04-22T15:12:04.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 5213 17.43 MB/sec warmup 13 sec latency 18.779 ms 2024-04-22T15:12:05.422 INFO:teuthology.orchestra.run.smithi119.stdout: 5 5975 16.87 MB/sec warmup 14 sec latency 22.935 ms 2024-04-22T15:12:05.657 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:05 smithi119 ceph-mon[21835]: pgmap v244: 97 pgs: 97 active+clean; 126 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 12 MiB/s wr, 666 op/s; 1 B/s, 0 objects/s recovering 2024-04-22T15:12:06.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:05 smithi186 ceph-mon[24041]: pgmap v244: 97 pgs: 97 active+clean; 126 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 12 MiB/s wr, 666 op/s; 1 B/s, 0 objects/s recovering 2024-04-22T15:12:06.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 6512 17.52 MB/sec warmup 15 sec latency 53.044 ms 2024-04-22T15:12:06.585 INFO:teuthology.orchestra.run.smithi119.stderr:module 'nfs' is already enabled 2024-04-22T15:12:06.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:06 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/843653609' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-22T15:12:07.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/843653609' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-22T15:12:07.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 7263 18.15 MB/sec warmup 16 sec latency 24.604 ms 2024-04-22T15:12:07.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:07 smithi119 ceph-mon[21835]: pgmap v245: 97 pgs: 97 active+clean; 135 MiB data, 278 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 12 MiB/s wr, 810 op/s 2024-04-22T15:12:07.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:07 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/843653609' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-22T15:12:07.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:07 smithi119 ceph-mon[21835]: mgrmap e21: smithi119.jjbxxd(active, since 6m), standbys: smithi186.vtjgwg 2024-04-22T15:12:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:07 smithi186 ceph-mon[24041]: pgmap v245: 97 pgs: 97 active+clean; 135 MiB data, 278 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 12 MiB/s wr, 810 op/s 2024-04-22T15:12:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:07 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/843653609' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-22T15:12:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:07 smithi186 ceph-mon[24041]: mgrmap e21: smithi119.jjbxxd(active, since 6m), standbys: smithi186.vtjgwg 2024-04-22T15:12:08.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 7665 17.43 MB/sec warmup 17 sec latency 23.334 ms 2024-04-22T15:12:08.449 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-22T15:12:09.421 INFO:teuthology.orchestra.run.smithi119.stdout: 5 8158 16.87 MB/sec warmup 18 sec latency 53.251 ms 2024-04-22T15:12:09.731 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:09 smithi119 ceph-mon[21835]: pgmap v246: 97 pgs: 97 active+clean; 142 MiB data, 300 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 12 MiB/s wr, 803 op/s 2024-04-22T15:12:10.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:09 smithi186 ceph-mon[24041]: pgmap v246: 97 pgs: 97 active+clean; 142 MiB data, 300 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 12 MiB/s wr, 803 op/s 2024-04-22T15:12:10.422 INFO:teuthology.orchestra.run.smithi119.stdout: 5 8792 16.96 MB/sec warmup 19 sec latency 23.579 ms 2024-04-22T15:12:11.356 INFO:teuthology.orchestra.run.smithi119.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:12:11.422 INFO:teuthology.orchestra.run.smithi119.stdout: 5 9487 16.55 MB/sec warmup 20 sec latency 21.251 ms 2024-04-22T15:12:11.687 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:11 smithi186 ceph-mon[24041]: pgmap v247: 97 pgs: 97 active+clean; 148 MiB data, 325 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 11 MiB/s wr, 864 op/s 2024-04-22T15:12:11.687 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:12:11.687 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:12:11.687 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:12:11.687 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:11 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:12:11.819 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:11 smithi119 ceph-mon[21835]: pgmap v247: 97 pgs: 97 active+clean; 148 MiB data, 325 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 11 MiB/s wr, 864 op/s 2024-04-22T15:12:11.819 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:12:11.820 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:12:11.820 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:12:11.820 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:11 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:12:12.422 INFO:teuthology.orchestra.run.smithi119.stdout: 5 9928 16.74 MB/sec warmup 21 sec latency 28.404 ms 2024-04-22T15:12:12.559 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-22T15:12:12.559 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T15:12:12.559 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-22T15:12:12.834 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:12 smithi119 ceph-mon[21835]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:12.835 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:12 smithi119 ceph-mon[21835]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:12:12.835 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:12 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:12:12.835 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:12 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:12:13.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:12 smithi186 ceph-mon[24041]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:13.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:12 smithi186 ceph-mon[24041]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:12:13.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:12 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:12:13.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:12 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:12:13.422 INFO:teuthology.orchestra.run.smithi119.stdout: 5 10616 17.18 MB/sec warmup 22 sec latency 21.757 ms 2024-04-22T15:12:13.588 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:13 smithi119 ceph-mon[21835]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:12:13.588 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:13 smithi119 ceph-mon[21835]: pgmap v248: 97 pgs: 97 active+clean; 146 MiB data, 369 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 779 op/s 2024-04-22T15:12:13.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:13 smithi186 ceph-mon[24041]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:12:13.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:13 smithi186 ceph-mon[24041]: pgmap v248: 97 pgs: 97 active+clean; 146 MiB data, 369 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 779 op/s 2024-04-22T15:12:14.425 INFO:teuthology.orchestra.run.smithi119.stdout: 5 11026 16.99 MB/sec warmup 23 sec latency 23.296 ms 2024-04-22T15:12:15.429 INFO:teuthology.orchestra.run.smithi119.stdout: 5 11556 16.46 MB/sec warmup 24 sec latency 21.593 ms 2024-04-22T15:12:15.602 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:15 smithi119 ceph-mon[21835]: pgmap v249: 97 pgs: 97 active+clean; 144 MiB data, 397 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 861 op/s 2024-04-22T15:12:15.981 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:15 smithi186 ceph-mon[24041]: pgmap v249: 97 pgs: 97 active+clean; 144 MiB data, 397 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 861 op/s 2024-04-22T15:12:16.075 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:12:16.429 INFO:teuthology.orchestra.run.smithi119.stdout: 5 12040 16.44 MB/sec warmup 25 sec latency 30.991 ms 2024-04-22T15:12:17.131 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:12:17.131 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (5m) 28s ago 6m 23.5M - ba2b418f427c 0e23a591f269 2024-04-22T15:12:17.131 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (6m) 28s ago 6m 6405k - 17.2.0 e1d6a67b021e 37eb54a66662 2024-04-22T15:12:17.131 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (6m) 30s ago 6m 6405k - 17.2.0 e1d6a67b021e ee37991f5d20 2024-04-22T15:12:17.131 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (5m) 28s ago 6m 52.1M - 8.3.5 dad864ee21e9 a8f43d2ae83a 2024-04-22T15:12:17.131 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (58s) 28s ago 57s 13.6M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:12:17.131 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (55s) 30s ago 55s 12.1M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:9283 running (8m) 28s ago 8m 460M - 17.2.0 e1d6a67b021e bac90840f90b 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283 running (6m) 30s ago 6m 412M - 17.2.0 e1d6a67b021e b36d3b07de5e 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (8m) 28s ago 8m 46.9M 2048M 17.2.0 e1d6a67b021e e31b6591d879 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (6m) 30s ago 6m 40.2M 2048M 17.2.0 e1d6a67b021e a804e52c2c5e 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (38s) 28s ago 38s 47.0M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (33s) 30s ago 32s 46.7M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (6m) 28s ago 6m 18.8M - 1dbe0e931976 2499dd8265c7 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (6m) 30s ago 5m 16.6M - 1dbe0e931976 5a18b815fb1f 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (5m) 28s ago 5m 43.5M 3216M 17.2.0 e1d6a67b021e 9999e9684eba 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (4m) 28s ago 4m 45.9M 3216M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (4m) 28s ago 4m 40.2M 3216M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (4m) 28s ago 4m 39.8M 3216M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (3m) 30s ago 3m 40.1M 3984M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:12:17.132 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (3m) 30s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:12:17.133 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (2m) 30s ago 2m 40.3M 3984M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:12:17.133 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (2m) 30s ago 2m 40.2M 3984M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:12:17.133 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (5m) 28s ago 5m 64.1M - 514e6a882f6e b8d42c7e1876 2024-04-22T15:12:17.429 INFO:teuthology.orchestra.run.smithi119.stdout: 5 12720 16.14 MB/sec warmup 26 sec latency 44.398 ms 2024-04-22T15:12:17.616 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:17 smithi119 ceph-mon[21835]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:17.616 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:17 smithi119 ceph-mon[21835]: pgmap v250: 97 pgs: 97 active+clean; 149 MiB data, 415 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 827 op/s 2024-04-22T15:12:17.617 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:17 smithi119 ceph-mon[21835]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:17.779 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:12:17.779 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:12:17.779 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:12:17.779 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:12:17.779 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:12:17.779 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:12:17.779 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:12:17.779 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:12:17.779 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:12:17.779 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:12:17.779 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:12:17.779 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:12:17.780 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:12:17.780 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:12:17.780 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-22T15:12:17.780 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:12:17.780 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:12:18.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:17 smithi186 ceph-mon[24041]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:18.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:17 smithi186 ceph-mon[24041]: pgmap v250: 97 pgs: 97 active+clean; 149 MiB data, 415 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 827 op/s 2024-04-22T15:12:18.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:17 smithi186 ceph-mon[24041]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:18.324 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:12:18.324 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", 2024-04-22T15:12:18.324 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:12:18.324 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [], 2024-04-22T15:12:18.324 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "", 2024-04-22T15:12:18.325 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa image" 2024-04-22T15:12:18.325 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:12:18.429 INFO:teuthology.orchestra.run.smithi119.stdout: 5 13192 15.94 MB/sec warmup 27 sec latency 22.985 ms 2024-04-22T15:12:18.623 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:18 smithi119 ceph-mon[21835]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:18.624 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:18 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/202504496' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:12:18.962 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:12:19.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:18 smithi186 ceph-mon[24041]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:19.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:18 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/202504496' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:12:19.429 INFO:teuthology.orchestra.run.smithi119.stdout: 5 13884 16.45 MB/sec warmup 28 sec latency 21.832 ms 2024-04-22T15:12:19.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:19 smithi119 ceph-mon[21835]: pgmap v251: 97 pgs: 97 active+clean; 154 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 8.3 MiB/s wr, 788 op/s 2024-04-22T15:12:19.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:19 smithi119 ceph-mon[21835]: from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:19.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:19 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/873057478' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:12:20.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:19 smithi186 ceph-mon[24041]: pgmap v251: 97 pgs: 97 active+clean; 154 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 8.3 MiB/s wr, 788 op/s 2024-04-22T15:12:20.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:19 smithi186 ceph-mon[24041]: from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:20.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:19 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/873057478' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:12:20.429 INFO:teuthology.orchestra.run.smithi119.stdout: 5 14462 16.74 MB/sec warmup 29 sec latency 17.528 ms 2024-04-22T15:12:21.429 INFO:teuthology.orchestra.run.smithi119.stdout: 5 15000 16.31 MB/sec warmup 30 sec latency 26.528 ms 2024-04-22T15:12:21.642 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:21 smithi119 ceph-mon[21835]: pgmap v252: 97 pgs: 97 active+clean; 160 MiB data, 449 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.2 MiB/s wr, 795 op/s 2024-04-22T15:12:22.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:21 smithi186 ceph-mon[24041]: pgmap v252: 97 pgs: 97 active+clean; 160 MiB data, 449 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.2 MiB/s wr, 795 op/s 2024-04-22T15:12:22.430 INFO:teuthology.orchestra.run.smithi119.stdout: 5 15740 16.54 MB/sec warmup 31 sec latency 13.883 ms 2024-04-22T15:12:23.429 INFO:teuthology.orchestra.run.smithi119.stdout: 5 16475 16.23 MB/sec warmup 32 sec latency 27.977 ms 2024-04-22T15:12:23.653 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:23 smithi119 ceph-mon[21835]: pgmap v253: 97 pgs: 97 active+clean; 164 MiB data, 481 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 7.1 MiB/s wr, 692 op/s 2024-04-22T15:12:23.916 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:23 smithi186 ceph-mon[24041]: pgmap v253: 97 pgs: 97 active+clean; 164 MiB data, 481 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 7.1 MiB/s wr, 692 op/s 2024-04-22T15:12:24.430 INFO:teuthology.orchestra.run.smithi119.stdout: 5 17347 16.79 MB/sec warmup 33 sec latency 17.670 ms 2024-04-22T15:12:25.430 INFO:teuthology.orchestra.run.smithi119.stdout: 5 18080 17.12 MB/sec warmup 34 sec latency 12.583 ms 2024-04-22T15:12:25.662 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:25 smithi119 ceph-mon[21835]: pgmap v254: 97 pgs: 97 active+clean; 171 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.7 MiB/s wr, 805 op/s 2024-04-22T15:12:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:25 smithi186 ceph-mon[24041]: pgmap v254: 97 pgs: 97 active+clean; 171 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.7 MiB/s wr, 805 op/s 2024-04-22T15:12:26.430 INFO:teuthology.orchestra.run.smithi119.stdout: 5 18778 16.86 MB/sec warmup 35 sec latency 15.954 ms 2024-04-22T15:12:27.430 INFO:teuthology.orchestra.run.smithi119.stdout: 5 19776 16.96 MB/sec warmup 36 sec latency 12.995 ms 2024-04-22T15:12:27.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:27 smithi119 ceph-mon[21835]: pgmap v255: 97 pgs: 97 active+clean; 178 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 9.2 MiB/s wr, 916 op/s 2024-04-22T15:12:28.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:27 smithi186 ceph-mon[24041]: pgmap v255: 97 pgs: 97 active+clean; 178 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 9.2 MiB/s wr, 916 op/s 2024-04-22T15:12:28.430 INFO:teuthology.orchestra.run.smithi119.stdout: 5 20839 17.44 MB/sec warmup 37 sec latency 15.698 ms 2024-04-22T15:12:29.431 INFO:teuthology.orchestra.run.smithi119.stdout: 5 21524 17.75 MB/sec warmup 38 sec latency 23.784 ms 2024-04-22T15:12:29.812 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:29 smithi119 ceph-mon[21835]: pgmap v256: 97 pgs: 97 active+clean; 182 MiB data, 632 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.6 MiB/s wr, 928 op/s 2024-04-22T15:12:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:29 smithi186 ceph-mon[24041]: pgmap v256: 97 pgs: 97 active+clean; 182 MiB data, 632 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.6 MiB/s wr, 928 op/s 2024-04-22T15:12:30.431 INFO:teuthology.orchestra.run.smithi119.stdout: 5 21965 17.43 MB/sec warmup 39 sec latency 22.456 ms 2024-04-22T15:12:31.431 INFO:teuthology.orchestra.run.smithi119.stdout: 5 22778 17.46 MB/sec warmup 40 sec latency 19.510 ms 2024-04-22T15:12:31.739 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:31 smithi119 ceph-mon[21835]: pgmap v257: 97 pgs: 97 active+clean; 182 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.9 MiB/s wr, 871 op/s 2024-04-22T15:12:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:31 smithi186 ceph-mon[24041]: pgmap v257: 97 pgs: 97 active+clean; 182 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.9 MiB/s wr, 871 op/s 2024-04-22T15:12:32.431 INFO:teuthology.orchestra.run.smithi119.stdout: 5 23687 17.39 MB/sec warmup 41 sec latency 14.788 ms 2024-04-22T15:12:33.431 INFO:teuthology.orchestra.run.smithi119.stdout: 5 24512 17.78 MB/sec warmup 42 sec latency 27.859 ms 2024-04-22T15:12:33.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:33 smithi119 ceph-mon[21835]: pgmap v258: 97 pgs: 97 active+clean; 192 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 9.9 MiB/s wr, 931 op/s 2024-04-22T15:12:33.916 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:33 smithi186 ceph-mon[24041]: pgmap v258: 97 pgs: 97 active+clean; 192 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 9.9 MiB/s wr, 931 op/s 2024-04-22T15:12:34.431 INFO:teuthology.orchestra.run.smithi119.stdout: 5 25228 17.99 MB/sec warmup 43 sec latency 21.138 ms 2024-04-22T15:12:35.431 INFO:teuthology.orchestra.run.smithi119.stdout: 5 25878 17.75 MB/sec warmup 44 sec latency 24.621 ms 2024-04-22T15:12:35.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:35 smithi119 ceph-mon[21835]: pgmap v259: 97 pgs: 97 active+clean; 195 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 915 op/s 2024-04-22T15:12:36.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:35 smithi186 ceph-mon[24041]: pgmap v259: 97 pgs: 97 active+clean; 195 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 915 op/s 2024-04-22T15:12:36.432 INFO:teuthology.orchestra.run.smithi119.stdout: 5 26617 17.79 MB/sec warmup 45 sec latency 20.207 ms 2024-04-22T15:12:37.433 INFO:teuthology.orchestra.run.smithi119.stdout: 5 27317 17.63 MB/sec warmup 46 sec latency 21.425 ms 2024-04-22T15:12:37.631 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:37 smithi119 ceph-mon[21835]: pgmap v260: 97 pgs: 97 active+clean; 204 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-04-22T15:12:38.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:37 smithi186 ceph-mon[24041]: pgmap v260: 97 pgs: 97 active+clean; 204 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-04-22T15:12:38.432 INFO:teuthology.orchestra.run.smithi119.stdout: 5 28381 18.21 MB/sec warmup 47 sec latency 20.653 ms 2024-04-22T15:12:39.432 INFO:teuthology.orchestra.run.smithi119.stdout: 5 29041 18.14 MB/sec warmup 48 sec latency 15.667 ms 2024-04-22T15:12:39.641 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:39 smithi119 ceph-mon[21835]: pgmap v261: 97 pgs: 97 active+clean; 202 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 950 op/s 2024-04-22T15:12:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:39 smithi186 ceph-mon[24041]: pgmap v261: 97 pgs: 97 active+clean; 202 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 950 op/s 2024-04-22T15:12:40.433 INFO:teuthology.orchestra.run.smithi119.stdout: 5 29762 18.14 MB/sec warmup 49 sec latency 34.566 ms 2024-04-22T15:12:41.433 INFO:teuthology.orchestra.run.smithi119.stdout: 5 30616 18.04 MB/sec warmup 50 sec latency 22.740 ms 2024-04-22T15:12:41.653 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:41 smithi119 ceph-mon[21835]: pgmap v262: 97 pgs: 97 active+clean; 205 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 895 op/s 2024-04-22T15:12:42.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:41 smithi186 ceph-mon[24041]: pgmap v262: 97 pgs: 97 active+clean; 205 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 895 op/s 2024-04-22T15:12:42.433 INFO:teuthology.orchestra.run.smithi119.stdout: 5 31369 18.17 MB/sec warmup 51 sec latency 25.518 ms 2024-04-22T15:12:43.433 INFO:teuthology.orchestra.run.smithi119.stdout: 5 32076 18.39 MB/sec warmup 52 sec latency 51.587 ms 2024-04-22T15:12:43.663 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:43 smithi119 ceph-mon[21835]: pgmap v263: 97 pgs: 97 active+clean; 212 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 10 MiB/s wr, 1.03k op/s 2024-04-22T15:12:43.916 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:43 smithi186 ceph-mon[24041]: pgmap v263: 97 pgs: 97 active+clean; 212 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 10 MiB/s wr, 1.03k op/s 2024-04-22T15:12:44.433 INFO:teuthology.orchestra.run.smithi119.stdout: 5 32677 18.25 MB/sec warmup 53 sec latency 22.676 ms 2024-04-22T15:12:45.435 INFO:teuthology.orchestra.run.smithi119.stdout: 5 33441 18.24 MB/sec warmup 54 sec latency 25.730 ms 2024-04-22T15:12:45.788 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:45 smithi119 ceph-mon[21835]: pgmap v264: 97 pgs: 97 active+clean; 211 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 893 op/s 2024-04-22T15:12:46.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:45 smithi186 ceph-mon[24041]: pgmap v264: 97 pgs: 97 active+clean; 211 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 893 op/s 2024-04-22T15:12:46.435 INFO:teuthology.orchestra.run.smithi119.stdout: 5 34321 18.18 MB/sec warmup 55 sec latency 13.404 ms 2024-04-22T15:12:47.438 INFO:teuthology.orchestra.run.smithi119.stdout: 5 35161 18.47 MB/sec warmup 56 sec latency 30.857 ms 2024-04-22T15:12:47.801 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:47 smithi119 ceph-mon[21835]: pgmap v265: 97 pgs: 97 active+clean; 216 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 997 op/s 2024-04-22T15:12:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:47 smithi186 ceph-mon[24041]: pgmap v265: 97 pgs: 97 active+clean; 216 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 997 op/s 2024-04-22T15:12:48.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 35835 18.56 MB/sec warmup 57 sec latency 27.614 ms 2024-04-22T15:12:49.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 36359 18.35 MB/sec warmup 58 sec latency 25.088 ms 2024-04-22T15:12:49.486 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:12:49.825 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:49 smithi119 ceph-mon[21835]: pgmap v266: 97 pgs: 97 active+clean; 225 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 961 op/s 2024-04-22T15:12:49.935 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:49 smithi186 ceph-mon[24041]: pgmap v266: 97 pgs: 97 active+clean; 225 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 961 op/s 2024-04-22T15:12:50.352 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:12:50.352 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (6m) 61s ago 7m 23.5M - ba2b418f427c 0e23a591f269 2024-04-22T15:12:50.352 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (7m) 61s ago 7m 6405k - 17.2.0 e1d6a67b021e 37eb54a66662 2024-04-22T15:12:50.352 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (6m) 63s ago 6m 6405k - 17.2.0 e1d6a67b021e ee37991f5d20 2024-04-22T15:12:50.352 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (5m) 61s ago 7m 52.1M - 8.3.5 dad864ee21e9 a8f43d2ae83a 2024-04-22T15:12:50.352 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (91s) 61s ago 90s 13.6M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:12:50.352 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (88s) 63s ago 88s 12.1M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:12:50.352 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:9283 running (8m) 61s ago 8m 460M - 17.2.0 e1d6a67b021e bac90840f90b 2024-04-22T15:12:50.352 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283 running (6m) 63s ago 6m 412M - 17.2.0 e1d6a67b021e b36d3b07de5e 2024-04-22T15:12:50.352 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (8m) 61s ago 8m 46.9M 2048M 17.2.0 e1d6a67b021e e31b6591d879 2024-04-22T15:12:50.352 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (6m) 63s ago 6m 40.2M 2048M 17.2.0 e1d6a67b021e a804e52c2c5e 2024-04-22T15:12:50.353 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (71s) 61s ago 71s 47.0M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:12:50.353 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (66s) 63s ago 66s 46.7M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:12:50.353 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (7m) 61s ago 7m 18.8M - 1dbe0e931976 2499dd8265c7 2024-04-22T15:12:50.353 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (6m) 63s ago 6m 16.6M - 1dbe0e931976 5a18b815fb1f 2024-04-22T15:12:50.353 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (5m) 61s ago 5m 43.5M 3216M 17.2.0 e1d6a67b021e 9999e9684eba 2024-04-22T15:12:50.353 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (5m) 61s ago 5m 45.9M 3216M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:12:50.353 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (5m) 61s ago 5m 40.2M 3216M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:12:50.353 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (4m) 61s ago 4m 39.8M 3216M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:12:50.353 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (4m) 63s ago 4m 40.1M 3984M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:12:50.353 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (3m) 63s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:12:50.354 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (3m) 63s ago 3m 40.3M 3984M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:12:50.354 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (3m) 63s ago 3m 40.2M 3984M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:12:50.354 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (6m) 61s ago 6m 64.1M - 514e6a882f6e b8d42c7e1876 2024-04-22T15:12:50.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 36820 18.27 MB/sec warmup 59 sec latency 36.807 ms 2024-04-22T15:12:50.836 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:50 smithi119 ceph-mon[21835]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:50.836 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:12:50.836 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:12:50.836 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:12:50.836 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:12:50.836 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:12:50.955 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:12:50.955 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:12:50.955 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:12:50.955 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:12:50.955 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:12:50.955 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:12:50.955 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:12:50.955 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:12:50.955 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:12:50.955 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:12:50.955 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:12:50.956 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:12:50.956 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:12:50.956 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:12:50.956 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-22T15:12:50.956 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:12:50.956 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:12:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:50 smithi186 ceph-mon[24041]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:12:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:12:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:12:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:12:51.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:12:51.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 37573 18.17 MB/sec warmup 60 sec latency 21.711 ms 2024-04-22T15:12:51.459 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:12:51.459 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:12:51.459 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:12:51.459 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [], 2024-04-22T15:12:51.459 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "0/23 daemons upgraded", 2024-04-22T15:12:51.459 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb image on host smithi186" 2024-04-22T15:12:51.459 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:12:51.839 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:51 smithi119 ceph-mon[21835]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:51.839 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:51 smithi119 ceph-mon[21835]: Upgrade: Target is version 19.0.0-3223-g5883d494 (unknown) 2024-04-22T15:12:51.840 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:51 smithi119 ceph-mon[21835]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:12:51.840 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:51 smithi119 ceph-mon[21835]: Upgrade: Need to upgrade myself (mgr.smithi119.jjbxxd) 2024-04-22T15:12:51.840 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:51 smithi119 ceph-mon[21835]: pgmap v267: 97 pgs: 97 active+clean; 232 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 982 op/s 2024-04-22T15:12:51.840 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:51 smithi119 ceph-mon[21835]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb on smithi186 2024-04-22T15:12:51.840 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:51 smithi119 ceph-mon[21835]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:51.840 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:51 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2677320354' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:12:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:51 smithi186 ceph-mon[24041]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:51 smithi186 ceph-mon[24041]: Upgrade: Target is version 19.0.0-3223-g5883d494 (unknown) 2024-04-22T15:12:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:51 smithi186 ceph-mon[24041]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:12:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:51 smithi186 ceph-mon[24041]: Upgrade: Need to upgrade myself (mgr.smithi119.jjbxxd) 2024-04-22T15:12:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:51 smithi186 ceph-mon[24041]: pgmap v267: 97 pgs: 97 active+clean; 232 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 982 op/s 2024-04-22T15:12:52.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:51 smithi186 ceph-mon[24041]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb on smithi186 2024-04-22T15:12:52.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:51 smithi186 ceph-mon[24041]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:52.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:51 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2677320354' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:12:52.088 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:12:52.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 38327 18.28 MB/sec warmup 61 sec latency 23.231 ms 2024-04-22T15:12:52.846 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:52 smithi119 ceph-mon[21835]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:52.846 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:52 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/15556732' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:12:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:52 smithi186 ceph-mon[24041]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:12:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:52 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/15556732' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:12:53.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 39226 18.59 MB/sec warmup 62 sec latency 13.286 ms 2024-04-22T15:12:53.852 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:53 smithi119 ceph-mon[21835]: pgmap v268: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-04-22T15:12:53.916 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:53 smithi186 ceph-mon[24041]: pgmap v268: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-04-22T15:12:54.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 39914 18.42 MB/sec warmup 63 sec latency 25.309 ms 2024-04-22T15:12:55.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 40641 18.39 MB/sec warmup 64 sec latency 19.491 ms 2024-04-22T15:12:55.861 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:55 smithi119 ceph-mon[21835]: pgmap v269: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 906 op/s 2024-04-22T15:12:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:55 smithi186 ceph-mon[24041]: pgmap v269: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 906 op/s 2024-04-22T15:12:56.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 41429 18.36 MB/sec warmup 65 sec latency 28.559 ms 2024-04-22T15:12:57.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 42221 18.54 MB/sec warmup 66 sec latency 19.095 ms 2024-04-22T15:12:57.822 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:57 smithi119 ceph-mon[21835]: pgmap v270: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 986 op/s 2024-04-22T15:12:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:57 smithi186 ceph-mon[24041]: pgmap v270: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 986 op/s 2024-04-22T15:12:58.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 43000 18.68 MB/sec warmup 67 sec latency 18.605 ms 2024-04-22T15:12:59.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 43803 18.58 MB/sec warmup 68 sec latency 21.010 ms 2024-04-22T15:12:59.794 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:12:59 smithi119 ceph-mon[21835]: pgmap v271: 97 pgs: 97 active+clean; 250 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-04-22T15:13:00.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:12:59 smithi186 ceph-mon[24041]: pgmap v271: 97 pgs: 97 active+clean; 250 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-04-22T15:13:00.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 44398 18.55 MB/sec warmup 69 sec latency 22.611 ms 2024-04-22T15:13:01.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 44973 18.41 MB/sec warmup 70 sec latency 31.541 ms 2024-04-22T15:13:01.821 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:01 smithi119 ceph-mon[21835]: pgmap v272: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 905 op/s 2024-04-22T15:13:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:01 smithi186 ceph-mon[24041]: pgmap v272: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 905 op/s 2024-04-22T15:13:02.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 45420 18.37 MB/sec warmup 71 sec latency 26.768 ms 2024-04-22T15:13:03.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 46184 18.55 MB/sec warmup 72 sec latency 19.024 ms 2024-04-22T15:13:03.840 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:03 smithi119 ceph-mon[21835]: pgmap v273: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 9.8 MiB/s wr, 949 op/s 2024-04-22T15:13:04.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:03 smithi186 ceph-mon[24041]: pgmap v273: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 9.8 MiB/s wr, 949 op/s 2024-04-22T15:13:04.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 46622 18.46 MB/sec warmup 73 sec latency 24.662 ms 2024-04-22T15:13:05.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 47165 18.32 MB/sec warmup 74 sec latency 37.188 ms 2024-04-22T15:13:05.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:05 smithi186 ceph-mon[24041]: pgmap v274: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 926 op/s 2024-04-22T15:13:05.876 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:05 smithi119 ceph-mon[21835]: pgmap v274: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 926 op/s 2024-04-22T15:13:06.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 47892 18.34 MB/sec warmup 75 sec latency 24.781 ms 2024-04-22T15:13:07.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 48705 18.28 MB/sec warmup 76 sec latency 28.455 ms 2024-04-22T15:13:07.887 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:07 smithi119 ceph-mon[21835]: pgmap v275: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 989 op/s 2024-04-22T15:13:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:07 smithi186 ceph-mon[24041]: pgmap v275: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 989 op/s 2024-04-22T15:13:08.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 49381 18.43 MB/sec warmup 77 sec latency 48.820 ms 2024-04-22T15:13:09.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 50043 18.49 MB/sec warmup 78 sec latency 25.114 ms 2024-04-22T15:13:09.916 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:09 smithi119 ceph-mon[21835]: pgmap v276: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 948 op/s 2024-04-22T15:13:10.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:09 smithi186 ceph-mon[24041]: pgmap v276: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 948 op/s 2024-04-22T15:13:10.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 50727 18.37 MB/sec warmup 79 sec latency 31.661 ms 2024-04-22T15:13:11.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 51561 18.40 MB/sec warmup 80 sec latency 23.604 ms 2024-04-22T15:13:11.926 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:11 smithi119 ceph-mon[21835]: pgmap v277: 97 pgs: 97 active+clean; 275 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 841 op/s 2024-04-22T15:13:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:11 smithi186 ceph-mon[24041]: pgmap v277: 97 pgs: 97 active+clean; 275 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 841 op/s 2024-04-22T15:13:12.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 52446 18.41 MB/sec warmup 81 sec latency 19.488 ms 2024-04-22T15:13:13.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 53337 18.62 MB/sec warmup 82 sec latency 27.295 ms 2024-04-22T15:13:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:13 smithi119 ceph-mon[21835]: pgmap v278: 97 pgs: 97 active+clean; 285 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.5 MiB/s wr, 1.02k op/s 2024-04-22T15:13:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:13 smithi186 ceph-mon[24041]: pgmap v278: 97 pgs: 97 active+clean; 285 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.5 MiB/s wr, 1.02k op/s 2024-04-22T15:13:14.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 53869 18.57 MB/sec warmup 83 sec latency 22.179 ms 2024-04-22T15:13:15.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 54558 18.46 MB/sec warmup 84 sec latency 35.300 ms 2024-04-22T15:13:15.704 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:15 smithi119 ceph-mon[21835]: pgmap v279: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 9.7 MiB/s wr, 893 op/s 2024-04-22T15:13:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:15 smithi186 ceph-mon[24041]: pgmap v279: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 9.7 MiB/s wr, 893 op/s 2024-04-22T15:13:16.439 INFO:teuthology.orchestra.run.smithi119.stdout: 5 55334 18.50 MB/sec warmup 85 sec latency 26.640 ms 2024-04-22T15:13:17.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 56105 18.56 MB/sec warmup 86 sec latency 28.803 ms 2024-04-22T15:13:17.775 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:17 smithi119 ceph-mon[21835]: pgmap v280: 97 pgs: 97 active+clean; 293 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 10 MiB/s wr, 960 op/s 2024-04-22T15:13:18.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:17 smithi186 ceph-mon[24041]: pgmap v280: 97 pgs: 97 active+clean; 293 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 10 MiB/s wr, 960 op/s 2024-04-22T15:13:18.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 56916 18.66 MB/sec warmup 87 sec latency 29.550 ms 2024-04-22T15:13:19.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 57507 18.63 MB/sec warmup 88 sec latency 40.147 ms 2024-04-22T15:13:19.788 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:19 smithi119 ceph-mon[21835]: pgmap v281: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-04-22T15:13:20.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:19 smithi186 ceph-mon[24041]: pgmap v281: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-04-22T15:13:20.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 58147 18.54 MB/sec warmup 89 sec latency 109.543 ms 2024-04-22T15:13:21.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 59111 18.58 MB/sec warmup 90 sec latency 123.934 ms 2024-04-22T15:13:21.797 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:21 smithi119 ceph-mon[21835]: pgmap v282: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 946 op/s 2024-04-22T15:13:22.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:21 smithi186 ceph-mon[24041]: pgmap v282: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 946 op/s 2024-04-22T15:13:22.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 59770 18.62 MB/sec warmup 91 sec latency 37.281 ms 2024-04-22T15:13:22.574 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:13:23.439 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:13:23.439 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (6m) 94s ago 8m 23.5M - ba2b418f427c 0e23a591f269 2024-04-22T15:13:23.439 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (7m) 94s ago 7m 6405k - 17.2.0 e1d6a67b021e 37eb54a66662 2024-04-22T15:13:23.439 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (7m) 96s ago 7m 6405k - 17.2.0 e1d6a67b021e ee37991f5d20 2024-04-22T15:13:23.439 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (6m) 94s ago 7m 52.1M - 8.3.5 dad864ee21e9 a8f43d2ae83a 2024-04-22T15:13:23.439 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (2m) 94s ago 2m 13.6M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (2m) 96s ago 2m 12.1M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:9283 running (9m) 94s ago 9m 460M - 17.2.0 e1d6a67b021e bac90840f90b 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283 running (7m) 96s ago 7m 412M - 17.2.0 e1d6a67b021e b36d3b07de5e 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (9m) 94s ago 9m 46.9M 2048M 17.2.0 e1d6a67b021e e31b6591d879 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (7m) 96s ago 7m 40.2M 2048M 17.2.0 e1d6a67b021e a804e52c2c5e 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (104s) 94s ago 104s 47.0M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (99s) 96s ago 99s 46.7M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (7m) 94s ago 7m 18.8M - 1dbe0e931976 2499dd8265c7 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (7m) 96s ago 7m 16.6M - 1dbe0e931976 5a18b815fb1f 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (6m) 94s ago 6m 43.5M 3216M 17.2.0 e1d6a67b021e 9999e9684eba 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (6m) 94s ago 5m 45.9M 3216M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (5m) 94s ago 5m 40.2M 3216M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:13:23.440 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (5m) 94s ago 5m 39.8M 3216M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:13:23.441 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (4m) 96s ago 4m 40.1M 3984M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:13:23.441 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (4m) 96s ago 4m 39.1M 3984M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:13:23.441 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (3m) 96s ago 3m 40.3M 3984M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:13:23.441 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (3m) 96s ago 3m 40.2M 3984M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:13:23.441 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (6m) 94s ago 6m 64.1M - 514e6a882f6e b8d42c7e1876 2024-04-22T15:13:23.441 INFO:teuthology.orchestra.run.smithi119.stdout: 5 60394 18.67 MB/sec warmup 92 sec latency 31.626 ms 2024-04-22T15:13:23.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:23 smithi119 ceph-mon[21835]: pgmap v283: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-04-22T15:13:23.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:23 smithi119 ceph-mon[21835]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:13:24.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:23 smithi186 ceph-mon[24041]: pgmap v283: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-04-22T15:13:24.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:23 smithi186 ceph-mon[24041]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:13:24.040 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:13:24.041 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-22T15:13:24.042 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:13:24.042 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:13:24.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 60827 18.62 MB/sec warmup 93 sec latency 22.485 ms 2024-04-22T15:13:24.510 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:13:24.510 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:13:24.510 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:13:24.510 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [], 2024-04-22T15:13:24.510 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "0/23 daemons upgraded", 2024-04-22T15:13:24.510 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb image on host smithi186" 2024-04-22T15:13:24.510 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:13:24.843 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:24 smithi119 ceph-mon[21835]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:13:24.843 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:24 smithi119 ceph-mon[21835]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:13:24.844 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:24 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/1823054630' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:13:25.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:24 smithi186 ceph-mon[24041]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:13:25.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:24 smithi186 ceph-mon[24041]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:13:25.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:24 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/1823054630' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:13:25.085 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:13:25.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 61367 18.49 MB/sec warmup 94 sec latency 24.875 ms 2024-04-22T15:13:25.858 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:25 smithi119 ceph-mon[21835]: pgmap v284: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 892 op/s 2024-04-22T15:13:25.858 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:25 smithi119 ceph-mon[21835]: from='client.14678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:13:25.858 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:25 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3501805238' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:13:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:25 smithi186 ceph-mon[24041]: pgmap v284: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 892 op/s 2024-04-22T15:13:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:25 smithi186 ceph-mon[24041]: from='client.14678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:13:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:25 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/3501805238' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:13:26.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 61826 18.42 MB/sec warmup 95 sec latency 50.468 ms 2024-04-22T15:13:27.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 62403 18.38 MB/sec warmup 96 sec latency 33.985 ms 2024-04-22T15:13:27.900 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:27 smithi119 ceph-mon[21835]: pgmap v285: 97 pgs: 97 active+clean; 314 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-22T15:13:28.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:27 smithi186 ceph-mon[24041]: pgmap v285: 97 pgs: 97 active+clean; 314 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-22T15:13:28.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 62959 18.31 MB/sec warmup 97 sec latency 44.567 ms 2024-04-22T15:13:29.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 63497 18.32 MB/sec warmup 98 sec latency 51.642 ms 2024-04-22T15:13:29.931 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:29 smithi119 ceph-mon[21835]: pgmap v286: 97 pgs: 97 active+clean; 318 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 9.8 MiB/s wr, 923 op/s 2024-04-22T15:13:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:29 smithi186 ceph-mon[24041]: pgmap v286: 97 pgs: 97 active+clean; 318 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 9.8 MiB/s wr, 923 op/s 2024-04-22T15:13:30.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 63952 18.31 MB/sec warmup 99 sec latency 95.068 ms 2024-04-22T15:13:31.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 64357 18.27 MB/sec warmup 100 sec latency 47.865 ms 2024-04-22T15:13:31.965 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:31 smithi119 ceph-mon[21835]: pgmap v287: 97 pgs: 97 active+clean; 317 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 800 op/s 2024-04-22T15:13:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:31 smithi186 ceph-mon[24041]: pgmap v287: 97 pgs: 97 active+clean; 317 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 800 op/s 2024-04-22T15:13:32.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 64834 18.15 MB/sec warmup 101 sec latency 43.652 ms 2024-04-22T15:13:32.970 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:32 smithi119 ceph-mon[21835]: Upgrade: Updating mgr.smithi186.vtjgwg 2024-04-22T15:13:32.970 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:32 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:32.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:32 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi186.vtjgwg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:13:32.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:32 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:13:32.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:32 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:13:32.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:32 smithi119 ceph-mon[21835]: Deploying daemon mgr.smithi186.vtjgwg on smithi186 2024-04-22T15:13:32.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:32 smithi119 ceph-mon[21835]: pgmap v288: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 921 op/s 2024-04-22T15:13:33.247 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:32 smithi186 ceph-mon[24041]: Upgrade: Updating mgr.smithi186.vtjgwg 2024-04-22T15:13:33.247 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:32 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:33.247 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:32 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi186.vtjgwg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:13:33.247 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:32 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:13:33.247 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:32 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:13:33.247 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:32 smithi186 ceph-mon[24041]: Deploying daemon mgr.smithi186.vtjgwg on smithi186 2024-04-22T15:13:33.248 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:32 smithi186 ceph-mon[24041]: pgmap v288: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 921 op/s 2024-04-22T15:13:33.440 INFO:teuthology.orchestra.run.smithi119.stdout: 5 65399 18.10 MB/sec warmup 102 sec latency 24.171 ms 2024-04-22T15:13:34.448 INFO:teuthology.orchestra.run.smithi119.stdout: 5 66046 18.07 MB/sec warmup 103 sec latency 29.318 ms 2024-04-22T15:13:35.397 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:35 smithi186 ceph-mon[24041]: pgmap v289: 97 pgs: 97 active+clean; 329 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.6 MiB/s wr, 836 op/s 2024-04-22T15:13:35.404 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:35 smithi119 ceph-mon[21835]: pgmap v289: 97 pgs: 97 active+clean; 329 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.6 MiB/s wr, 836 op/s 2024-04-22T15:13:35.449 INFO:teuthology.orchestra.run.smithi119.stdout: 5 66641 18.04 MB/sec warmup 104 sec latency 34.128 ms 2024-04-22T15:13:36.450 INFO:teuthology.orchestra.run.smithi119.stdout: 5 67157 18.07 MB/sec warmup 105 sec latency 26.359 ms 2024-04-22T15:13:37.431 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:37 smithi119 ceph-mon[21835]: pgmap v290: 97 pgs: 97 active+clean; 325 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.8 MiB/s wr, 811 op/s 2024-04-22T15:13:37.431 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:37.432 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:13:37.432 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:13:37.432 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:37 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:13:37.449 INFO:teuthology.orchestra.run.smithi119.stdout: 5 67745 18.10 MB/sec warmup 106 sec latency 63.176 ms 2024-04-22T15:13:37.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:37 smithi186 ceph-mon[24041]: pgmap v290: 97 pgs: 97 active+clean; 325 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.8 MiB/s wr, 811 op/s 2024-04-22T15:13:37.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:37.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:13:37.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:13:37.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:37 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:13:38.450 INFO:teuthology.orchestra.run.smithi119.stdout: 5 68240 18.01 MB/sec warmup 107 sec latency 29.061 ms 2024-04-22T15:13:39.318 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:39 smithi186 ceph-mon[24041]: pgmap v291: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.4 MiB/s wr, 748 op/s 2024-04-22T15:13:39.392 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:39 smithi119 ceph-mon[21835]: pgmap v291: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.4 MiB/s wr, 748 op/s 2024-04-22T15:13:39.449 INFO:teuthology.orchestra.run.smithi119.stdout: 5 68676 17.90 MB/sec warmup 108 sec latency 73.899 ms 2024-04-22T15:13:40.450 INFO:teuthology.orchestra.run.smithi119.stdout: 5 68984 17.83 MB/sec warmup 109 sec latency 124.010 ms 2024-04-22T15:13:41.250 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:41 smithi119 ceph-mon[21835]: pgmap v292: 97 pgs: 97 active+clean; 326 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.4 MiB/s wr, 690 op/s 2024-04-22T15:13:41.450 INFO:teuthology.orchestra.run.smithi119.stdout: 5 69508 17.78 MB/sec warmup 110 sec latency 104.842 ms 2024-04-22T15:13:41.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:41 smithi186 ceph-mon[24041]: pgmap v292: 97 pgs: 97 active+clean; 326 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.4 MiB/s wr, 690 op/s 2024-04-22T15:13:42.450 INFO:teuthology.orchestra.run.smithi119.stdout: 5 69944 17.68 MB/sec warmup 111 sec latency 117.237 ms 2024-04-22T15:13:43.450 INFO:teuthology.orchestra.run.smithi119.stdout: 5 70349 17.67 MB/sec warmup 112 sec latency 60.850 ms 2024-04-22T15:13:43.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:43 smithi119 ceph-mon[21835]: pgmap v293: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.6 MiB/s wr, 840 op/s 2024-04-22T15:13:43.995 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:43 smithi186 ceph-mon[24041]: pgmap v293: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.6 MiB/s wr, 840 op/s 2024-04-22T15:13:44.450 INFO:teuthology.orchestra.run.smithi119.stdout: 5 71011 17.73 MB/sec warmup 113 sec latency 40.281 ms 2024-04-22T15:13:44.643 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:44 smithi119 ceph-mon[21835]: Standby manager daemon smithi186.vtjgwg restarted 2024-04-22T15:13:44.643 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:44 smithi119 ceph-mon[21835]: Standby manager daemon smithi186.vtjgwg started 2024-04-22T15:13:44.644 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:44 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/crt"}]: dispatch 2024-04-22T15:13:44.644 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:44 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:13:44.644 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:44 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/key"}]: dispatch 2024-04-22T15:13:44.644 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:44 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:13:45.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:44 smithi186 ceph-mon[24041]: Standby manager daemon smithi186.vtjgwg restarted 2024-04-22T15:13:45.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:44 smithi186 ceph-mon[24041]: Standby manager daemon smithi186.vtjgwg started 2024-04-22T15:13:45.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:44 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/crt"}]: dispatch 2024-04-22T15:13:45.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:44 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:13:45.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:44 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/key"}]: dispatch 2024-04-22T15:13:45.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:44 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:13:45.450 INFO:teuthology.orchestra.run.smithi119.stdout: 5 71574 17.71 MB/sec warmup 114 sec latency 19.490 ms 2024-04-22T15:13:45.613 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:45 smithi119 ceph-mon[21835]: pgmap v294: 97 pgs: 97 active+clean; 321 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 701 op/s 2024-04-22T15:13:45.613 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:45 smithi119 ceph-mon[21835]: mgrmap e22: smithi119.jjbxxd(active, since 7m), standbys: smithi186.vtjgwg 2024-04-22T15:13:46.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:45 smithi186 ceph-mon[24041]: pgmap v294: 97 pgs: 97 active+clean; 321 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 701 op/s 2024-04-22T15:13:46.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:45 smithi186 ceph-mon[24041]: mgrmap e22: smithi119.jjbxxd(active, since 7m), standbys: smithi186.vtjgwg 2024-04-22T15:13:46.450 INFO:teuthology.orchestra.run.smithi119.stdout: 5 72205 17.63 MB/sec warmup 115 sec latency 26.940 ms 2024-04-22T15:13:47.454 INFO:teuthology.orchestra.run.smithi119.stdout: 5 72878 17.64 MB/sec warmup 116 sec latency 40.074 ms 2024-04-22T15:13:47.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:47 smithi119 ceph-mon[21835]: pgmap v295: 97 pgs: 97 active+clean; 315 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 726 op/s 2024-04-22T15:13:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:47 smithi186 ceph-mon[24041]: pgmap v295: 97 pgs: 97 active+clean; 315 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 726 op/s 2024-04-22T15:13:48.453 INFO:teuthology.orchestra.run.smithi119.stdout: 5 73509 17.59 MB/sec warmup 117 sec latency 113.618 ms 2024-04-22T15:13:49.453 INFO:teuthology.orchestra.run.smithi119.stdout: 5 74145 17.64 MB/sec warmup 118 sec latency 23.049 ms 2024-04-22T15:13:49.761 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:49 smithi119 ceph-mon[21835]: pgmap v296: 97 pgs: 97 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 760 op/s 2024-04-22T15:13:49.917 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:49 smithi186 ceph-mon[24041]: pgmap v296: 97 pgs: 97 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 760 op/s 2024-04-22T15:13:50.453 INFO:teuthology.orchestra.run.smithi119.stdout: 5 74677 17.67 MB/sec warmup 119 sec latency 38.821 ms 2024-04-22T15:13:50.773 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:13:50.773 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:13:50.774 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:50 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:50.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:13:50.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:13:50.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:50 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:51.799 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:51 smithi119 ceph-mon[21835]: pgmap v297: 97 pgs: 97 active+clean; 320 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 7.8 MiB/s wr, 755 op/s 2024-04-22T15:13:51.799 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:51 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:52.004 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:51 smithi186 ceph-mon[24041]: pgmap v297: 97 pgs: 97 active+clean; 320 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 7.8 MiB/s wr, 755 op/s 2024-04-22T15:13:52.004 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:51 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:52.453 INFO:teuthology.orchestra.run.smithi119.stdout: 5 75654 8.70 MB/sec execute 1 sec latency 35.308 ms 2024-04-22T15:13:53.453 INFO:teuthology.orchestra.run.smithi119.stdout: 5 76375 12.43 MB/sec execute 2 sec latency 20.196 ms 2024-04-22T15:13:53.615 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:53 smithi119 ceph-mon[21835]: pgmap v298: 97 pgs: 97 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.0 MiB/s wr, 899 op/s 2024-04-22T15:13:53.900 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:53 smithi186 ceph-mon[24041]: pgmap v298: 97 pgs: 97 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.0 MiB/s wr, 899 op/s 2024-04-22T15:13:54.454 INFO:teuthology.orchestra.run.smithi119.stdout: 5 76998 12.94 MB/sec execute 3 sec latency 30.320 ms 2024-04-22T15:13:55.454 INFO:teuthology.orchestra.run.smithi119.stdout: 5 77480 14.36 MB/sec execute 4 sec latency 39.691 ms 2024-04-22T15:13:55.585 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:13:55.836 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:55 smithi119 ceph-mon[21835]: pgmap v299: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 8.0 MiB/s wr, 742 op/s 2024-04-22T15:13:55.837 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:55 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:55.837 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:55 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:55.837 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:55 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:55.837 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:55 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:13:55.837 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:55 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr fail", "who": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:13:55.837 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:55 smithi119 ceph-mon[21835]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T15:13:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:55 smithi186 ceph-mon[24041]: pgmap v299: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 8.0 MiB/s wr, 742 op/s 2024-04-22T15:13:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:55 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:55 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:55 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:13:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:55 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:13:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:55 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr fail", "who": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:13:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:55 smithi186 ceph-mon[24041]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T15:13:56.454 INFO:teuthology.orchestra.run.smithi119.stdout: 5 78021 15.79 MB/sec execute 5 sec latency 26.713 ms 2024-04-22T15:13:56.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: Upgrade: Need to upgrade myself (mgr.smithi119.jjbxxd) 2024-04-22T15:13:56.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: Failing over to other MGR 2024-04-22T15:13:56.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:13:56.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: Standby manager daemon smithi119.jjbxxd started 2024-04-22T15:13:56.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "mgr fail", "who": "smithi119.jjbxxd"}]': finished 2024-04-22T15:13:56.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: mgrmap e23: smithi186.vtjgwg(active, starting, since 1.01449s), standbys: smithi119.jjbxxd 2024-04-22T15:13:56.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:13:56.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:13:56.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:13:56.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:13:56.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mgr metadata", "who": "smithi186.vtjgwg", "id": "smithi186.vtjgwg"}]: dispatch 2024-04-22T15:13:56.627 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mgr metadata", "who": "smithi119.jjbxxd", "id": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: Manager daemon smithi186.vtjgwg is now available 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:13:56.628 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:13:56.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:13:56.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi186.vtjgwg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:13:56.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi186.vtjgwg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:13:56.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi186.vtjgwg/trash_purge_schedule"}]: dispatch 2024-04-22T15:13:56.629 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:56 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi186.vtjgwg/trash_purge_schedule"}]: dispatch 2024-04-22T15:13:56.657 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: Upgrade: Need to upgrade myself (mgr.smithi119.jjbxxd) 2024-04-22T15:13:56.657 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: Failing over to other MGR 2024-04-22T15:13:56.657 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:13:56.657 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: Standby manager daemon smithi119.jjbxxd started 2024-04-22T15:13:56.657 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.14204 172.21.15.119:0/2476505232' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "mgr fail", "who": "smithi119.jjbxxd"}]': finished 2024-04-22T15:13:56.657 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: mgrmap e23: smithi186.vtjgwg(active, starting, since 1.01449s), standbys: smithi119.jjbxxd 2024-04-22T15:13:56.657 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mgr metadata", "who": "smithi186.vtjgwg", "id": "smithi186.vtjgwg"}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mgr metadata", "who": "smithi119.jjbxxd", "id": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:13:56.658 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:13:56.659 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:13:56.659 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:13:56.659 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:13:56.659 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: Manager daemon smithi186.vtjgwg is now available 2024-04-22T15:13:56.659 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:13:56.659 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:13:56.659 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:13:56.659 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi186.vtjgwg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:13:56.659 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi186.vtjgwg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:13:56.659 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi186.vtjgwg/trash_purge_schedule"}]: dispatch 2024-04-22T15:13:56.659 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:56 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi186.vtjgwg/trash_purge_schedule"}]: dispatch 2024-04-22T15:13:57.449 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:13:57.450 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (7m) 2m ago 8m 23.5M - ba2b418f427c 0e23a591f269 2024-04-22T15:13:57.450 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (8m) 2m ago 8m 6405k - 17.2.0 e1d6a67b021e 37eb54a66662 2024-04-22T15:13:57.450 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (7m) 7s ago 7m 6405k - 17.2.0 e1d6a67b021e ee37991f5d20 2024-04-22T15:13:57.450 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (6m) 2m ago 8m 52.1M - 8.3.5 dad864ee21e9 a8f43d2ae83a 2024-04-22T15:13:57.450 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (2m) 2m ago 2m 13.6M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:13:57.450 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (2m) 7s ago 2m 12.5M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:13:57.450 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:9283 running (9m) 2m ago 9m 460M - 17.2.0 e1d6a67b021e bac90840f90b 2024-04-22T15:13:57.450 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283 running (22s) 7s ago 7m 84.7M - 19.0.0-3223-g5883d494 3d983c26e278 fb27c8a37962 2024-04-22T15:13:57.450 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (9m) 2m ago 10m 46.9M 2048M 17.2.0 e1d6a67b021e e31b6591d879 2024-04-22T15:13:57.450 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (7m) 7s ago 7m 40.8M 2048M 17.2.0 e1d6a67b021e a804e52c2c5e 2024-04-22T15:13:57.450 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (2m) 2m ago 2m 47.0M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:13:57.450 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (2m) 7s ago 2m 73.2M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:13:57.451 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (8m) 2m ago 8m 18.8M - 1dbe0e931976 2499dd8265c7 2024-04-22T15:13:57.451 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (7m) 7s ago 7m 21.5M - 1dbe0e931976 5a18b815fb1f 2024-04-22T15:13:57.451 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (7m) 2m ago 7m 43.5M 3216M 17.2.0 e1d6a67b021e 9999e9684eba 2024-04-22T15:13:57.451 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (6m) 2m ago 6m 45.9M 3216M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:13:57.451 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (6m) 2m ago 6m 40.2M 3216M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:13:57.451 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (5m) 2m ago 5m 39.8M 3216M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:13:57.451 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (5m) 7s ago 5m 240M 3984M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:13:57.451 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (4m) 7s ago 4m 283M 3984M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:13:57.451 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (4m) 7s ago 4m 245M 3984M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:13:57.451 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (4m) 7s ago 4m 354M 3984M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:13:57.451 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (7m) 2m ago 7m 64.1M - 514e6a882f6e b8d42c7e1876 2024-04-22T15:13:57.453 INFO:teuthology.orchestra.run.smithi119.stdout: 5 78448 15.34 MB/sec execute 6 sec latency 37.211 ms 2024-04-22T15:13:58.085 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:13:58.085 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:13:58.085 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:13:58.085 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:13:58.086 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:13:58.086 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T15:13:58.086 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:13:58.086 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:13:58.086 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:13:58.086 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:13:58.086 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:13:58.086 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:13:58.086 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:13:58.086 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:13:58.086 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:13:58.086 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T15:13:58.087 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:13:58.087 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:13:58.087 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:13:58.150 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:58 smithi119 ceph-mon[21835]: Deploying cephadm binary to smithi186 2024-04-22T15:13:58.150 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:58 smithi119 ceph-mon[21835]: Deploying cephadm binary to smithi119 2024-04-22T15:13:58.150 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:58 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:13:58.150 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:58 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:13:58.150 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:58 smithi119 ceph-mon[21835]: [22/Apr/2024:15:13:57] ENGINE Bus STARTING 2024-04-22T15:13:58.150 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:58 smithi119 ceph-mon[21835]: mgrmap e24: smithi186.vtjgwg(active, since 2s), standbys: smithi119.jjbxxd 2024-04-22T15:13:58.150 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:58 smithi119 ceph-mon[21835]: pgmap v3: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:13:58.151 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:58 smithi119 ceph-mon[21835]: from='client.14696 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:13:58.151 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:58 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/3019735318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:13:58.454 INFO:teuthology.orchestra.run.smithi119.stdout: 5 78795 13.88 MB/sec execute 7 sec latency 39.001 ms 2024-04-22T15:13:58.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:58 smithi186 ceph-mon[24041]: Deploying cephadm binary to smithi186 2024-04-22T15:13:58.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:58 smithi186 ceph-mon[24041]: Deploying cephadm binary to smithi119 2024-04-22T15:13:58.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:58 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:13:58.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:58 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:13:58.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:58 smithi186 ceph-mon[24041]: [22/Apr/2024:15:13:57] ENGINE Bus STARTING 2024-04-22T15:13:58.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:58 smithi186 ceph-mon[24041]: mgrmap e24: smithi186.vtjgwg(active, since 2s), standbys: smithi119.jjbxxd 2024-04-22T15:13:58.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:58 smithi186 ceph-mon[24041]: pgmap v3: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:13:58.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:58 smithi186 ceph-mon[24041]: from='client.14696 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:13:58.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:58 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/3019735318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:13:58.718 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:13:58.718 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:13:58.718 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:13:58.718 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:13:58.719 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [], 2024-04-22T15:13:58.719 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "1/23 daemons upgraded", 2024-04-22T15:13:58.719 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "", 2024-04-22T15:13:58.719 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:13:58.719 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:13:59.427 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:59 smithi119 ceph-mon[21835]: [22/Apr/2024:15:13:57] ENGINE Serving on http://172.21.15.186:8765 2024-04-22T15:13:59.428 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:59 smithi119 ceph-mon[21835]: [22/Apr/2024:15:13:57] ENGINE Serving on https://172.21.15.186:7150 2024-04-22T15:13:59.428 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:59 smithi119 ceph-mon[21835]: [22/Apr/2024:15:13:57] ENGINE Bus STARTED 2024-04-22T15:13:59.428 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:59 smithi119 ceph-mon[21835]: pgmap v4: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:13:59.428 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:13:59 smithi119 ceph-mon[21835]: mgrmap e25: smithi186.vtjgwg(active, since 3s), standbys: smithi119.jjbxxd 2024-04-22T15:13:59.437 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:13:59.454 INFO:teuthology.orchestra.run.smithi119.stdout: 5 79206 12.98 MB/sec execute 8 sec latency 44.625 ms 2024-04-22T15:13:59.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:59 smithi186 ceph-mon[24041]: [22/Apr/2024:15:13:57] ENGINE Serving on http://172.21.15.186:8765 2024-04-22T15:13:59.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:59 smithi186 ceph-mon[24041]: [22/Apr/2024:15:13:57] ENGINE Serving on https://172.21.15.186:7150 2024-04-22T15:13:59.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:59 smithi186 ceph-mon[24041]: [22/Apr/2024:15:13:57] ENGINE Bus STARTED 2024-04-22T15:13:59.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:59 smithi186 ceph-mon[24041]: pgmap v4: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:13:59.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:13:59 smithi186 ceph-mon[24041]: mgrmap e25: smithi186.vtjgwg(active, since 3s), standbys: smithi119.jjbxxd 2024-04-22T15:14:00.454 INFO:teuthology.orchestra.run.smithi119.stdout: 5 79630 12.79 MB/sec execute 9 sec latency 48.031 ms 2024-04-22T15:14:00.468 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:00 smithi119 ceph-mon[21835]: from='client.24459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:00.468 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:00 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2773528485' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:14:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:00 smithi186 ceph-mon[24041]: from='client.24459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:00 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2773528485' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:14:01.454 INFO:teuthology.orchestra.run.smithi119.stdout: 5 80247 12.98 MB/sec execute 10 sec latency 28.163 ms 2024-04-22T15:14:01.472 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:01 smithi119 ceph-mon[21835]: pgmap v5: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:14:01.472 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:01 smithi119 ceph-mon[21835]: mgrmap e26: smithi186.vtjgwg(active, since 5s), standbys: smithi119.jjbxxd 2024-04-22T15:14:01.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:01 smithi186 ceph-mon[24041]: pgmap v5: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:14:01.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:01 smithi186 ceph-mon[24041]: mgrmap e26: smithi186.vtjgwg(active, since 5s), standbys: smithi119.jjbxxd 2024-04-22T15:14:02.454 INFO:teuthology.orchestra.run.smithi119.stdout: 5 80819 13.11 MB/sec execute 11 sec latency 32.932 ms 2024-04-22T15:14:02.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:02 smithi119 ceph-mon[21835]: pgmap v6: 97 pgs: 97 active+clean; 321 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.7 MiB/s wr, 299 op/s 2024-04-22T15:14:03.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:02 smithi186 ceph-mon[24041]: pgmap v6: 97 pgs: 97 active+clean; 321 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.7 MiB/s wr, 299 op/s 2024-04-22T15:14:03.454 INFO:teuthology.orchestra.run.smithi119.stdout: 5 81271 13.43 MB/sec execute 12 sec latency 28.286 ms 2024-04-22T15:14:04.455 INFO:teuthology.orchestra.run.smithi119.stdout: 5 81767 13.82 MB/sec execute 13 sec latency 30.519 ms 2024-04-22T15:14:04.517 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:04 smithi119 ceph-mon[21835]: pgmap v7: 97 pgs: 97 active+clean; 315 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 4.4 MiB/s wr, 504 op/s 2024-04-22T15:14:04.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:04 smithi186 ceph-mon[24041]: pgmap v7: 97 pgs: 97 active+clean; 315 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 4.4 MiB/s wr, 504 op/s 2024-04-22T15:14:05.455 INFO:teuthology.orchestra.run.smithi119.stdout: 5 82074 13.51 MB/sec execute 14 sec latency 102.254 ms 2024-04-22T15:14:05.533 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:05 smithi119 ceph-mon[21835]: Standby manager daemon smithi119.jjbxxd restarted 2024-04-22T15:14:05.533 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:05 smithi119 ceph-mon[21835]: Standby manager daemon smithi119.jjbxxd started 2024-04-22T15:14:05.533 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:05 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.119:0/1497885561' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi119.jjbxxd/crt"}]: dispatch 2024-04-22T15:14:05.534 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:05 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.119:0/1497885561' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:14:05.534 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:05 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.119:0/1497885561' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi119.jjbxxd/key"}]: dispatch 2024-04-22T15:14:05.534 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:05 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.119:0/1497885561' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:14:05.754 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:05 smithi186 ceph-mon[24041]: Standby manager daemon smithi119.jjbxxd restarted 2024-04-22T15:14:05.754 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:05 smithi186 ceph-mon[24041]: Standby manager daemon smithi119.jjbxxd started 2024-04-22T15:14:05.754 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:05 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.119:0/1497885561' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi119.jjbxxd/crt"}]: dispatch 2024-04-22T15:14:05.754 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:05 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.119:0/1497885561' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:14:05.755 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:05 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.119:0/1497885561' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi119.jjbxxd/key"}]: dispatch 2024-04-22T15:14:05.755 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:05 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.119:0/1497885561' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:14:06.455 INFO:teuthology.orchestra.run.smithi119.stdout: 5 82479 13.08 MB/sec execute 15 sec latency 34.952 ms 2024-04-22T15:14:06.540 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:06 smithi119 ceph-mon[21835]: mgrmap e27: smithi186.vtjgwg(active, since 10s), standbys: smithi119.jjbxxd 2024-04-22T15:14:06.541 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:06 smithi119 ceph-mon[21835]: pgmap v8: 97 pgs: 97 active+clean; 315 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.4 MiB/s wr, 392 op/s 2024-04-22T15:14:06.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:06 smithi186 ceph-mon[24041]: mgrmap e27: smithi186.vtjgwg(active, since 10s), standbys: smithi119.jjbxxd 2024-04-22T15:14:06.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:06 smithi186 ceph-mon[24041]: pgmap v8: 97 pgs: 97 active+clean; 315 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.4 MiB/s wr, 392 op/s 2024-04-22T15:14:07.455 INFO:teuthology.orchestra.run.smithi119.stdout: 5 82943 12.65 MB/sec execute 16 sec latency 45.716 ms 2024-04-22T15:14:08.303 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:08 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:08.303 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:08 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:08.455 INFO:teuthology.orchestra.run.smithi119.stdout: 5 83514 12.88 MB/sec execute 17 sec latency 26.859 ms 2024-04-22T15:14:08.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:08 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:08.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:08 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:09.455 INFO:teuthology.orchestra.run.smithi119.stdout: 5 84092 12.76 MB/sec execute 18 sec latency 50.003 ms 2024-04-22T15:14:09.568 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:09 smithi119 ceph-mon[21835]: pgmap v9: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.0 MiB/s wr, 609 op/s 2024-04-22T15:14:09.568 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:09 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:09.568 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:09 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:09.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:09 smithi186 ceph-mon[24041]: pgmap v9: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.0 MiB/s wr, 609 op/s 2024-04-22T15:14:09.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:09 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:09.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:09 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:10.455 INFO:teuthology.orchestra.run.smithi119.stdout: 5 84584 13.09 MB/sec execute 19 sec latency 34.446 ms 2024-04-22T15:14:11.455 INFO:teuthology.orchestra.run.smithi119.stdout: 5 85385 13.94 MB/sec execute 20 sec latency 25.461 ms 2024-04-22T15:14:11.588 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:11 smithi119 ceph-mon[21835]: pgmap v10: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.5 MiB/s wr, 558 op/s 2024-04-22T15:14:11.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:11 smithi186 ceph-mon[24041]: pgmap v10: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.5 MiB/s wr, 558 op/s 2024-04-22T15:14:12.455 INFO:teuthology.orchestra.run.smithi119.stdout: 5 85791 13.68 MB/sec execute 21 sec latency 32.653 ms 2024-04-22T15:14:12.594 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:12 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:14:12.595 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:12 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:12.595 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:12 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:12.595 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:12 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:12.595 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:12 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:12.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:12 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:14:12.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:12 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:12.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:12 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:12.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:12 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:12.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:12 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:13.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 86297 13.37 MB/sec execute 22 sec latency 37.678 ms 2024-04-22T15:14:13.604 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 7.9 MiB/s wr, 839 op/s 2024-04-22T15:14:13.604 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.604 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.604 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.604 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.604 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.604 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.604 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.604 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: Adjusting osd_memory_target on smithi186 to 2704M 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:13.605 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:14:13.606 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:13 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:14:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 7.9 MiB/s wr, 839 op/s 2024-04-22T15:14:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: Adjusting osd_memory_target on smithi186 to 2704M 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:14:13.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:13.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:14:13.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:13 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:14:14.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 86896 13.53 MB/sec execute 23 sec latency 30.843 ms 2024-04-22T15:14:14.613 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:14 smithi119 ceph-mon[21835]: Adjusting osd_memory_target on smithi119 to 1936M 2024-04-22T15:14:14.613 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:14 smithi119 ceph-mon[21835]: Updating smithi119:/etc/ceph/ceph.conf 2024-04-22T15:14:14.614 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:14 smithi119 ceph-mon[21835]: Updating smithi186:/etc/ceph/ceph.conf 2024-04-22T15:14:14.614 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:14 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:14.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:14 smithi186 ceph-mon[24041]: Adjusting osd_memory_target on smithi119 to 1936M 2024-04-22T15:14:14.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:14 smithi186 ceph-mon[24041]: Updating smithi119:/etc/ceph/ceph.conf 2024-04-22T15:14:14.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:14 smithi186 ceph-mon[24041]: Updating smithi186:/etc/ceph/ceph.conf 2024-04-22T15:14:14.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:14 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.455 INFO:teuthology.orchestra.run.smithi119.stdout: 5 87731 13.73 MB/sec execute 24 sec latency 29.623 ms 2024-04-22T15:14:15.582 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: Updating smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:14:15.582 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: Updating smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:14:15.582 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:14:15.582 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: Updating smithi119:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:14:15.582 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: Updating smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:14:15.582 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: Updating smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:14:15.582 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: pgmap v12: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 8.0 MiB/s wr, 772 op/s 2024-04-22T15:14:15.582 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.583 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.583 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.583 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.583 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.583 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.583 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.583 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.583 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:15 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: Updating smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:14:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: Updating smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:14:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:14:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: Updating smithi119:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:14:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: Updating smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:14:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: Updating smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:14:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: pgmap v12: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 8.0 MiB/s wr, 772 op/s 2024-04-22T15:14:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:15.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:15 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:16.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 88522 14.33 MB/sec execute 25 sec latency 43.820 ms 2024-04-22T15:14:16.623 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:16 smithi119 ceph-mon[21835]: Reconfiguring alertmanager.smithi119 (dependencies changed)... 2024-04-22T15:14:16.623 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:16 smithi119 ceph-mon[21835]: Reconfiguring daemon alertmanager.smithi119 on smithi119 2024-04-22T15:14:16.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:16 smithi186 ceph-mon[24041]: Reconfiguring alertmanager.smithi119 (dependencies changed)... 2024-04-22T15:14:16.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:16 smithi186 ceph-mon[24041]: Reconfiguring daemon alertmanager.smithi119 on smithi119 2024-04-22T15:14:17.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 89185 14.70 MB/sec execute 26 sec latency 23.779 ms 2024-04-22T15:14:17.574 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:17 smithi119 ceph-mon[21835]: pgmap v13: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.6 MiB/s wr, 603 op/s 2024-04-22T15:14:17.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:17 smithi186 ceph-mon[24041]: pgmap v13: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.6 MiB/s wr, 603 op/s 2024-04-22T15:14:18.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 89708 14.37 MB/sec execute 27 sec latency 60.667 ms 2024-04-22T15:14:18.593 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:18 smithi119 ceph-mon[21835]: pgmap v14: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 989 op/s 2024-04-22T15:14:18.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:18 smithi186 ceph-mon[24041]: pgmap v14: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 989 op/s 2024-04-22T15:14:19.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 90189 14.24 MB/sec execute 28 sec latency 25.405 ms 2024-04-22T15:14:20.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 90795 14.32 MB/sec execute 29 sec latency 49.572 ms 2024-04-22T15:14:20.618 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:20 smithi119 ceph-mon[21835]: pgmap v15: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.8 MiB/s wr, 725 op/s 2024-04-22T15:14:20.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:20 smithi186 ceph-mon[24041]: pgmap v15: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.8 MiB/s wr, 725 op/s 2024-04-22T15:14:21.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 91351 14.21 MB/sec execute 30 sec latency 28.066 ms 2024-04-22T15:14:22.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 91854 14.41 MB/sec execute 31 sec latency 46.608 ms 2024-04-22T15:14:22.645 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:22 smithi119 ceph-mon[21835]: pgmap v16: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.2 MiB/s wr, 875 op/s 2024-04-22T15:14:23.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:22 smithi186 ceph-mon[24041]: pgmap v16: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.2 MiB/s wr, 875 op/s 2024-04-22T15:14:23.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 92554 14.80 MB/sec execute 32 sec latency 24.098 ms 2024-04-22T15:14:24.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 93026 14.61 MB/sec execute 33 sec latency 27.018 ms 2024-04-22T15:14:24.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:24 smithi119 ceph-mon[21835]: pgmap v17: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 752 op/s 2024-04-22T15:14:24.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:24 smithi186 ceph-mon[24041]: pgmap v17: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 752 op/s 2024-04-22T15:14:25.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 93566 14.40 MB/sec execute 34 sec latency 23.718 ms 2024-04-22T15:14:25.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:25 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:25.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:25 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:25.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:25 smithi119 ceph-mon[21835]: Reconfiguring grafana.smithi119 (dependencies changed)... 2024-04-22T15:14:25.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:25 smithi119 ceph-mon[21835]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-22T15:14:25.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:25 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:25.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:25 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:25.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:25 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:14:25.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:25 smithi119 ceph-mon[21835]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:14:25.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:25 smithi119 ceph-mon[21835]: Reconfiguring daemon grafana.smithi119 on smithi119 2024-04-22T15:14:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:25 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:25 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:25 smithi186 ceph-mon[24041]: Reconfiguring grafana.smithi119 (dependencies changed)... 2024-04-22T15:14:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:25 smithi186 ceph-mon[24041]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-22T15:14:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:25 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:26.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:25 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:26.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:25 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:14:26.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:25 smithi186 ceph-mon[24041]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:14:26.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:25 smithi186 ceph-mon[24041]: Reconfiguring daemon grafana.smithi119 on smithi119 2024-04-22T15:14:26.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 94461 14.63 MB/sec execute 35 sec latency 15.346 ms 2024-04-22T15:14:26.925 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:26 smithi119 ceph-mon[21835]: pgmap v18: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 693 op/s 2024-04-22T15:14:26.926 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:26 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:14:27.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:26 smithi186 ceph-mon[24041]: pgmap v18: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 693 op/s 2024-04-22T15:14:27.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:26 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:14:27.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 95249 14.94 MB/sec execute 36 sec latency 22.614 ms 2024-04-22T15:14:28.456 INFO:teuthology.orchestra.run.smithi119.stdout: 5 95968 15.22 MB/sec execute 37 sec latency 22.521 ms 2024-04-22T15:14:28.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:28 smithi119 ceph-mon[21835]: pgmap v19: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.09k op/s 2024-04-22T15:14:28.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:28 smithi186 ceph-mon[24041]: pgmap v19: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.09k op/s 2024-04-22T15:14:29.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 96408 15.18 MB/sec execute 38 sec latency 35.130 ms 2024-04-22T15:14:29.953 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:14:30.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 96838 14.86 MB/sec execute 39 sec latency 50.465 ms 2024-04-22T15:14:30.704 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:30 smithi119 ceph-mon[21835]: from='client.24471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:30.704 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:30 smithi119 ceph-mon[21835]: pgmap v20: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 702 op/s 2024-04-22T15:14:30.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:30 smithi186 ceph-mon[24041]: from='client.24471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:30.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:30 smithi186 ceph-mon[24041]: pgmap v20: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 702 op/s 2024-04-22T15:14:31.056 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:14:31.056 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 starting - - - - 2024-04-22T15:14:31.056 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (9m) 19s ago 9m 6405k - 17.2.0 e1d6a67b021e 37eb54a66662 2024-04-22T15:14:31.056 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (8m) 23s ago 8m 6405k - 17.2.0 e1d6a67b021e ee37991f5d20 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (7m) 19s ago 8m 53.4M - 8.3.5 dad864ee21e9 a8f43d2ae83a 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (3m) 19s ago 3m 36.9M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (3m) 23s ago 3m 13.0M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:9283 running (10m) 19s ago 10m 322M - 17.2.0 e1d6a67b021e bac90840f90b 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283 running (56s) 23s ago 8m 494M - 19.0.0-3223-g5883d494 3d983c26e278 fb27c8a37962 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (10m) 19s ago 10m 51.8M 2048M 17.2.0 e1d6a67b021e e31b6591d879 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (8m) 23s ago 8m 43.5M 2048M 17.2.0 e1d6a67b021e a804e52c2c5e 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (2m) 19s ago 2m 237M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (2m) 23s ago 2m 76.0M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (8m) 19s ago 8m 22.8M - 1.3.1 1dbe0e931976 2499dd8265c7 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (8m) 23s ago 8m 16.8M - 1.3.1 1dbe0e931976 5a18b815fb1f 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (7m) 19s ago 7m 330M 1936M 17.2.0 e1d6a67b021e 9999e9684eba 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (7m) 19s ago 7m 261M 1936M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:14:31.057 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (6m) 19s ago 6m 204M 1936M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:14:31.058 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (6m) 19s ago 6m 277M 1936M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:14:31.058 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (5m) 23s ago 5m 301M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:14:31.058 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (5m) 23s ago 5m 326M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:14:31.058 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (5m) 23s ago 5m 268M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:14:31.058 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (4m) 23s ago 4m 396M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:14:31.058 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (7m) 19s ago 7m 68.4M - 2.33.4 514e6a882f6e b8d42c7e1876 2024-04-22T15:14:31.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 97203 14.67 MB/sec execute 40 sec latency 56.775 ms 2024-04-22T15:14:31.714 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:14:31.714 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:14:31.714 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:14:31.714 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:14:31.714 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:14:31.714 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:14:31.715 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:14:31.716 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:31 smithi119 ceph-mon[21835]: from='client.24475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:31.716 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:31 smithi119 ceph-mon[21835]: from='client.24479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:31.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:31 smithi186 ceph-mon[24041]: from='client.24475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:31.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:31 smithi186 ceph-mon[24041]: from='client.24479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:32.281 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:14:32.282 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:14:32.282 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:14:32.282 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:14:32.282 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [], 2024-04-22T15:14:32.282 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "1/23 daemons upgraded", 2024-04-22T15:14:32.282 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "", 2024-04-22T15:14:32.282 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:14:32.282 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:14:32.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 97982 14.84 MB/sec execute 41 sec latency 18.859 ms 2024-04-22T15:14:32.718 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:32 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2567121082' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:14:32.719 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:32 smithi119 ceph-mon[21835]: from='client.24487 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:32.719 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:32 smithi119 ceph-mon[21835]: pgmap v21: 97 pgs: 97 active+clean; 320 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.1 MiB/s wr, 760 op/s 2024-04-22T15:14:32.969 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:14:33.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:32 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2567121082' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:14:33.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:32 smithi186 ceph-mon[24041]: from='client.24487 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:33.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:32 smithi186 ceph-mon[24041]: pgmap v21: 97 pgs: 97 active+clean; 320 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.1 MiB/s wr, 760 op/s 2024-04-22T15:14:33.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 98504 14.76 MB/sec execute 42 sec latency 45.658 ms 2024-04-22T15:14:33.724 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:33 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:33.724 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:33 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:33.724 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:33 smithi119 ceph-mon[21835]: Reconfiguring node-exporter.smithi119 (dependencies changed)... 2024-04-22T15:14:33.725 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:33 smithi119 ceph-mon[21835]: Reconfiguring daemon node-exporter.smithi119 on smithi119 2024-04-22T15:14:33.725 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:33 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2453148225' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:14:34.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:33 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:34.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:33 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:34.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:33 smithi186 ceph-mon[24041]: Reconfiguring node-exporter.smithi119 (dependencies changed)... 2024-04-22T15:14:34.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:33 smithi186 ceph-mon[24041]: Reconfiguring daemon node-exporter.smithi119 on smithi119 2024-04-22T15:14:34.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:33 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2453148225' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:14:34.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 99130 14.97 MB/sec execute 43 sec latency 30.197 ms 2024-04-22T15:14:34.735 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:34 smithi119 ceph-mon[21835]: pgmap v22: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.8 MiB/s wr, 869 op/s 2024-04-22T15:14:35.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:34 smithi186 ceph-mon[24041]: pgmap v22: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.8 MiB/s wr, 869 op/s 2024-04-22T15:14:35.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 99634 15.04 MB/sec execute 44 sec latency 35.469 ms 2024-04-22T15:14:36.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 99984 14.95 MB/sec execute 45 sec latency 39.996 ms 2024-04-22T15:14:37.124 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:37 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:37.124 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:37 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:37.124 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:37 smithi119 ceph-mon[21835]: Reconfiguring prometheus.smithi119 (dependencies changed)... 2024-04-22T15:14:37.124 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:37 smithi119 ceph-mon[21835]: Reconfiguring daemon prometheus.smithi119 on smithi119 2024-04-22T15:14:37.124 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:37 smithi119 ceph-mon[21835]: pgmap v23: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 712 op/s 2024-04-22T15:14:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:37 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:37 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:37 smithi186 ceph-mon[24041]: Reconfiguring prometheus.smithi119 (dependencies changed)... 2024-04-22T15:14:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:37 smithi186 ceph-mon[24041]: Reconfiguring daemon prometheus.smithi119 on smithi119 2024-04-22T15:14:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:37 smithi186 ceph-mon[24041]: pgmap v23: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 712 op/s 2024-04-22T15:14:37.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 100384 14.68 MB/sec execute 46 sec latency 62.097 ms 2024-04-22T15:14:38.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 100748 14.53 MB/sec execute 47 sec latency 36.737 ms 2024-04-22T15:14:38.633 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:38 smithi119 ceph-mon[21835]: pgmap v24: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 1.00k op/s 2024-04-22T15:14:38.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:38 smithi186 ceph-mon[24041]: pgmap v24: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 1.00k op/s 2024-04-22T15:14:39.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 101452 14.66 MB/sec execute 48 sec latency 32.424 ms 2024-04-22T15:14:40.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 102035 14.59 MB/sec execute 49 sec latency 40.371 ms 2024-04-22T15:14:40.653 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:40 smithi119 ceph-mon[21835]: pgmap v25: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.4 MiB/s wr, 607 op/s 2024-04-22T15:14:40.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:40 smithi186 ceph-mon[24041]: pgmap v25: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.4 MiB/s wr, 607 op/s 2024-04-22T15:14:41.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 102428 14.62 MB/sec execute 50 sec latency 36.277 ms 2024-04-22T15:14:41.667 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:41 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:14:42.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:41 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:14:42.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 102861 14.64 MB/sec execute 51 sec latency 47.134 ms 2024-04-22T15:14:42.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:42 smithi119 ceph-mon[21835]: pgmap v26: 97 pgs: 97 active+clean; 316 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.9 MiB/s wr, 749 op/s 2024-04-22T15:14:43.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:42 smithi186 ceph-mon[24041]: pgmap v26: 97 pgs: 97 active+clean; 316 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.9 MiB/s wr, 749 op/s 2024-04-22T15:14:43.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 103224 14.63 MB/sec execute 52 sec latency 50.868 ms 2024-04-22T15:14:44.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 103549 14.50 MB/sec execute 53 sec latency 51.955 ms 2024-04-22T15:14:44.705 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:44 smithi119 ceph-mon[21835]: pgmap v27: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.4 MiB/s wr, 851 op/s 2024-04-22T15:14:44.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:44 smithi186 ceph-mon[24041]: pgmap v27: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.4 MiB/s wr, 851 op/s 2024-04-22T15:14:45.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 104001 14.30 MB/sec execute 54 sec latency 28.859 ms 2024-04-22T15:14:46.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 104325 14.16 MB/sec execute 55 sec latency 31.099 ms 2024-04-22T15:14:46.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:46 smithi119 ceph-mon[21835]: pgmap v28: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.3 MiB/s wr, 592 op/s 2024-04-22T15:14:46.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:46 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:46.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:46 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:46.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:46 smithi186 ceph-mon[24041]: pgmap v28: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.3 MiB/s wr, 592 op/s 2024-04-22T15:14:46.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:46 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:46.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:46 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:47.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 104701 14.17 MB/sec execute 56 sec latency 46.156 ms 2024-04-22T15:14:47.824 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:47 smithi119 ceph-mon[21835]: Reconfiguring node-exporter.smithi186 (dependencies changed)... 2024-04-22T15:14:47.824 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:47 smithi119 ceph-mon[21835]: Reconfiguring daemon node-exporter.smithi186 on smithi186 2024-04-22T15:14:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:47 smithi186 ceph-mon[24041]: Reconfiguring node-exporter.smithi186 (dependencies changed)... 2024-04-22T15:14:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:47 smithi186 ceph-mon[24041]: Reconfiguring daemon node-exporter.smithi186 on smithi186 2024-04-22T15:14:48.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 105317 14.12 MB/sec execute 57 sec latency 90.033 ms 2024-04-22T15:14:48.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:48 smithi186 ceph-mon[24041]: pgmap v29: 97 pgs: 97 active+clean; 316 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 830 op/s 2024-04-22T15:14:48.843 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:48 smithi119 ceph-mon[21835]: pgmap v29: 97 pgs: 97 active+clean; 316 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 830 op/s 2024-04-22T15:14:49.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 105765 14.08 MB/sec execute 58 sec latency 23.064 ms 2024-04-22T15:14:50.361 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:50.361 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:50.361 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:14:50.361 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:14:50.361 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi119.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T15:14:50.361 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi119.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T15:14:50.361 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:50.361 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:14:50.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:14:50.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi119.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T15:14:50.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi119.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T15:14:50.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:50.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:14:50.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:14:50.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi119.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T15:14:50.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi119.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T15:14:50.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:50.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:14:50.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: Upgrade: Need to upgrade myself (mgr.smithi186.vtjgwg) 2024-04-22T15:14:50.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:50 smithi119 ceph-mon[21835]: Upgrade: Need to upgrade myself (mgr.smithi186.vtjgwg) 2024-04-22T15:14:50.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 106274 14.17 MB/sec execute 59 sec latency 32.718 ms 2024-04-22T15:14:50.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:50.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:50.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:14:50.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:14:50.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi119.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T15:14:50.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi119.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T15:14:50.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:50.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:14:50.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:14:50.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi119.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T15:14:50.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi119.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T15:14:50.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:50.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:14:50.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:14:50.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi119.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T15:14:50.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi119.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T15:14:50.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:50.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:14:50.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: Upgrade: Need to upgrade myself (mgr.smithi186.vtjgwg) 2024-04-22T15:14:50.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:50 smithi186 ceph-mon[24041]: Upgrade: Need to upgrade myself (mgr.smithi186.vtjgwg) 2024-04-22T15:14:51.376 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:51 smithi119 ceph-mon[21835]: pgmap v30: 97 pgs: 97 active+clean; 316 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.4 MiB/s wr, 540 op/s 2024-04-22T15:14:51.457 INFO:teuthology.orchestra.run.smithi119.stdout: 5 106733 14.23 MB/sec execute 60 sec latency 38.642 ms 2024-04-22T15:14:51.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:51 smithi186 ceph-mon[24041]: pgmap v30: 97 pgs: 97 active+clean; 316 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.4 MiB/s wr, 540 op/s 2024-04-22T15:14:52.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 107071 14.17 MB/sec execute 61 sec latency 44.836 ms 2024-04-22T15:14:52.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:52 smithi119 ceph-mon[21835]: pgmap v31: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 733 op/s 2024-04-22T15:14:52.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:52 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:52.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:52 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi119.jjbxxd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:14:52.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:52 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi119.jjbxxd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:14:52.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:52 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:14:52.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:52 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:14:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:52 smithi186 ceph-mon[24041]: pgmap v31: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 733 op/s 2024-04-22T15:14:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:52 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:52 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi119.jjbxxd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:14:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:52 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi119.jjbxxd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:14:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:52 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:14:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:52 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:14:53.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 107428 13.97 MB/sec execute 62 sec latency 50.111 ms 2024-04-22T15:14:53.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:53 smithi119 ceph-mon[21835]: Upgrade: Updating mgr.smithi119.jjbxxd 2024-04-22T15:14:53.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:53 smithi119 ceph-mon[21835]: Deploying daemon mgr.smithi119.jjbxxd on smithi119 2024-04-22T15:14:54.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:53 smithi186 ceph-mon[24041]: Upgrade: Updating mgr.smithi119.jjbxxd 2024-04-22T15:14:54.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:53 smithi186 ceph-mon[24041]: Deploying daemon mgr.smithi119.jjbxxd on smithi119 2024-04-22T15:14:54.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 107730 13.84 MB/sec execute 63 sec latency 41.214 ms 2024-04-22T15:14:54.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:54 smithi119 ceph-mon[21835]: pgmap v32: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.0 MiB/s wr, 673 op/s 2024-04-22T15:14:55.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:54 smithi186 ceph-mon[24041]: pgmap v32: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.0 MiB/s wr, 673 op/s 2024-04-22T15:14:55.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 108086 13.84 MB/sec execute 64 sec latency 35.117 ms 2024-04-22T15:14:56.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 108532 13.77 MB/sec execute 65 sec latency 53.768 ms 2024-04-22T15:14:56.706 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:56 smithi119 ceph-mon[21835]: pgmap v33: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.3 MiB/s wr, 514 op/s 2024-04-22T15:14:56.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:56 smithi186 ceph-mon[24041]: pgmap v33: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.3 MiB/s wr, 514 op/s 2024-04-22T15:14:57.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 109135 13.73 MB/sec execute 66 sec latency 27.902 ms 2024-04-22T15:14:57.589 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:57 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:14:57.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:57 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:14:58.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 109725 13.87 MB/sec execute 67 sec latency 27.354 ms 2024-04-22T15:14:58.718 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:58 smithi119 ceph-mon[21835]: pgmap v34: 97 pgs: 97 active+clean; 321 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 763 op/s 2024-04-22T15:14:58.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:58 smithi186 ceph-mon[24041]: pgmap v34: 97 pgs: 97 active+clean; 321 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 763 op/s 2024-04-22T15:14:59.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 110288 13.96 MB/sec execute 68 sec latency 25.354 ms 2024-04-22T15:14:59.967 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:59 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:59.967 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:59 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:59.967 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:59 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:14:59.967 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:59 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:14:59.967 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:14:59 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:00.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:59 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:00.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:59 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:00.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:59 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:15:00.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:59 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:00.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:14:59 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:00.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 110708 13.91 MB/sec execute 69 sec latency 33.503 ms 2024-04-22T15:15:00.989 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:00 smithi119 ceph-mon[21835]: pgmap v35: 97 pgs: 97 active+clean; 321 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 525 op/s 2024-04-22T15:15:01.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:00 smithi186 ceph-mon[24041]: pgmap v35: 97 pgs: 97 active+clean; 321 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 525 op/s 2024-04-22T15:15:01.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 111159 13.78 MB/sec execute 70 sec latency 33.622 ms 2024-04-22T15:15:02.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 111587 13.80 MB/sec execute 71 sec latency 22.325 ms 2024-04-22T15:15:02.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:02 smithi119 ceph-mon[21835]: pgmap v36: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 795 op/s 2024-04-22T15:15:02.895 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:02 smithi186 ceph-mon[24041]: pgmap v36: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 795 op/s 2024-04-22T15:15:03.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 112036 13.76 MB/sec execute 72 sec latency 59.098 ms 2024-04-22T15:15:03.614 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:15:04.222 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:04 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:04.222 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:04 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:04.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 112494 13.67 MB/sec execute 73 sec latency 55.848 ms 2024-04-22T15:15:04.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:04 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:04.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:04 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:04.597 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:15:04.597 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 starting - - - - 2024-04-22T15:15:04.597 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (9m) 53s ago 9m 6405k - 17.2.0 e1d6a67b021e 37eb54a66662 2024-04-22T15:15:04.597 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (9m) 1s ago 9m 6405k - 17.2.0 e1d6a67b021e ee37991f5d20 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 starting - - - - 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (3m) 53s ago 3m 36.9M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (3m) 1s ago 3m 13.2M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 starting - - - - 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283 running (89s) 1s ago 9m 506M - 19.0.0-3223-g5883d494 3d983c26e278 fb27c8a37962 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (10m) 53s ago 11m 51.8M 2048M 17.2.0 e1d6a67b021e e31b6591d879 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (8m) 1s ago 8m 42.4M 2048M 17.2.0 e1d6a67b021e a804e52c2c5e 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (3m) 53s ago 3m 237M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (3m) 1s ago 3m 82.1M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 starting - - - - 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (15s) 1s ago 8m 9629k - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (8m) 53s ago 8m 330M 1936M 17.2.0 e1d6a67b021e 9999e9684eba 2024-04-22T15:15:04.598 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (7m) 53s ago 7m 261M 1936M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:15:04.599 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (7m) 53s ago 7m 204M 1936M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:15:04.599 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (6m) 53s ago 6m 277M 1936M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:15:04.599 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (6m) 1s ago 6m 370M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:15:04.599 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (6m) 1s ago 6m 418M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:15:04.599 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (5m) 1s ago 5m 345M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:15:04.599 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (5m) 1s ago 5m 496M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:15:04.599 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 starting - - - - 2024-04-22T15:15:05.226 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:15:05.226 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:15:05.227 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T15:15:05.228 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:15:05.228 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:15:05.228 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:15:05.246 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:05 smithi119 ceph-mon[21835]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:05.247 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:05 smithi119 ceph-mon[21835]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:05.247 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:05 smithi119 ceph-mon[21835]: pgmap v37: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.7 MiB/s wr, 631 op/s 2024-04-22T15:15:05.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 112759 13.56 MB/sec execute 74 sec latency 39.432 ms 2024-04-22T15:15:05.483 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:05 smithi186 ceph-mon[24041]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:05.483 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:05 smithi186 ceph-mon[24041]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:05.483 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:05 smithi186 ceph-mon[24041]: pgmap v37: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.7 MiB/s wr, 631 op/s 2024-04-22T15:15:05.666 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:15:05.666 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:15:05.666 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:15:05.666 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:15:05.666 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [], 2024-04-22T15:15:05.666 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "", 2024-04-22T15:15:05.666 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading mgr daemons", 2024-04-22T15:15:05.666 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:15:05.666 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:15:06.175 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:15:06.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:06 smithi119 ceph-mon[21835]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:06.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:06 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/998755729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:15:06.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 113099 13.58 MB/sec execute 75 sec latency 36.245 ms 2024-04-22T15:15:06.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:06 smithi186 ceph-mon[24041]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:06.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:06 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/998755729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:15:07.320 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:07 smithi119 ceph-mon[21835]: from='client.14764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:07.320 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:07 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2966386187' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:15:07.320 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:07 smithi119 ceph-mon[21835]: pgmap v38: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 548 op/s 2024-04-22T15:15:07.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 113514 13.59 MB/sec execute 76 sec latency 59.521 ms 2024-04-22T15:15:07.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:07 smithi186 ceph-mon[24041]: from='client.14764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:07.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:07 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2966386187' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:15:07.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:07 smithi186 ceph-mon[24041]: pgmap v38: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 548 op/s 2024-04-22T15:15:08.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 113869 13.61 MB/sec execute 77 sec latency 36.889 ms 2024-04-22T15:15:08.604 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:08 smithi119 ceph-mon[21835]: pgmap v39: 97 pgs: 97 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 775 op/s 2024-04-22T15:15:08.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:08 smithi186 ceph-mon[24041]: pgmap v39: 97 pgs: 97 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 775 op/s 2024-04-22T15:15:09.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 114121 13.52 MB/sec execute 78 sec latency 41.106 ms 2024-04-22T15:15:09.615 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:09 smithi119 ceph-mon[21835]: Standby manager daemon smithi119.jjbxxd restarted 2024-04-22T15:15:09.615 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:09 smithi119 ceph-mon[21835]: Standby manager daemon smithi119.jjbxxd started 2024-04-22T15:15:09.615 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:09 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi119.jjbxxd/crt"}]: dispatch 2024-04-22T15:15:09.615 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:09 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:15:09.615 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:09 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi119.jjbxxd/key"}]: dispatch 2024-04-22T15:15:09.615 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:09 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:15:09.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:09 smithi186 ceph-mon[24041]: Standby manager daemon smithi119.jjbxxd restarted 2024-04-22T15:15:09.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:09 smithi186 ceph-mon[24041]: Standby manager daemon smithi119.jjbxxd started 2024-04-22T15:15:09.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:09 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi119.jjbxxd/crt"}]: dispatch 2024-04-22T15:15:09.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:09 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:15:09.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:09 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi119.jjbxxd/key"}]: dispatch 2024-04-22T15:15:09.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:09 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:15:10.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 114505 13.38 MB/sec execute 79 sec latency 45.218 ms 2024-04-22T15:15:10.632 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:10 smithi119 ceph-mon[21835]: mgrmap e28: smithi186.vtjgwg(active, since 74s), standbys: smithi119.jjbxxd 2024-04-22T15:15:10.632 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:10 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:10.633 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:10 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:10.633 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:10 smithi119 ceph-mon[21835]: pgmap v40: 97 pgs: 97 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.7 MiB/s wr, 526 op/s 2024-04-22T15:15:10.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:10 smithi186 ceph-mon[24041]: mgrmap e28: smithi186.vtjgwg(active, since 74s), standbys: smithi119.jjbxxd 2024-04-22T15:15:10.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:10 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:10.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:10 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:10.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:10 smithi186 ceph-mon[24041]: pgmap v40: 97 pgs: 97 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.7 MiB/s wr, 526 op/s 2024-04-22T15:15:11.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 114786 13.28 MB/sec execute 80 sec latency 33.278 ms 2024-04-22T15:15:12.422 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:12 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:15:12.422 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:12 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:12.422 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:12 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:12.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 115120 13.26 MB/sec execute 81 sec latency 40.641 ms 2024-04-22T15:15:12.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:12 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:15:12.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:12 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:12.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:12 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:13.435 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:13 smithi119 ceph-mon[21835]: pgmap v41: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 576 op/s 2024-04-22T15:15:13.458 INFO:teuthology.orchestra.run.smithi119.stdout: 5 115622 13.25 MB/sec execute 82 sec latency 44.630 ms 2024-04-22T15:15:13.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:13 smithi186 ceph-mon[24041]: pgmap v41: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 576 op/s 2024-04-22T15:15:14.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 116162 13.20 MB/sec execute 83 sec latency 30.378 ms 2024-04-22T15:15:14.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:14 smithi119 ceph-mon[21835]: pgmap v42: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.3 MiB/s wr, 468 op/s 2024-04-22T15:15:14.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:14 smithi186 ceph-mon[24041]: pgmap v42: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.3 MiB/s wr, 468 op/s 2024-04-22T15:15:15.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 116509 13.20 MB/sec execute 84 sec latency 27.496 ms 2024-04-22T15:15:16.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 117045 13.27 MB/sec execute 85 sec latency 34.390 ms 2024-04-22T15:15:16.727 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:16 smithi119 ceph-mon[21835]: pgmap v43: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-04-22T15:15:16.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:16 smithi186 ceph-mon[24041]: pgmap v43: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-04-22T15:15:17.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 117492 13.31 MB/sec execute 86 sec latency 43.961 ms 2024-04-22T15:15:18.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 117917 13.24 MB/sec execute 87 sec latency 41.204 ms 2024-04-22T15:15:19.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:19.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:19.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:15:19.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:15:19.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:19.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:19.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:15:19.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: Upgrade: Need to upgrade myself (mgr.smithi186.vtjgwg) 2024-04-22T15:15:19.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: Upgrade: Need to upgrade myself (mgr.smithi186.vtjgwg) 2024-04-22T15:15:19.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: Failing over to other MGR 2024-04-22T15:15:19.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mgr fail", "who": "smithi186.vtjgwg"}]: dispatch 2024-04-22T15:15:19.009 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mgr fail", "who": "smithi186.vtjgwg"}]: dispatch 2024-04-22T15:15:19.009 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T15:15:19.009 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:18 smithi119 ceph-mon[21835]: pgmap v45: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.0 MiB/s wr, 578 op/s 2024-04-22T15:15:19.260 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:19.260 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:19.260 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:15:19.260 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:15:19.260 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:19.260 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' 2024-04-22T15:15:19.260 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:15:19.261 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: Upgrade: Need to upgrade myself (mgr.smithi186.vtjgwg) 2024-04-22T15:15:19.261 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: Upgrade: Need to upgrade myself (mgr.smithi186.vtjgwg) 2024-04-22T15:15:19.261 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: Failing over to other MGR 2024-04-22T15:15:19.261 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: from='mgr.24421 172.21.15.186:0/413097580' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mgr fail", "who": "smithi186.vtjgwg"}]: dispatch 2024-04-22T15:15:19.261 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "mgr fail", "who": "smithi186.vtjgwg"}]: dispatch 2024-04-22T15:15:19.261 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T15:15:19.261 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:18 smithi186 ceph-mon[24041]: pgmap v45: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.0 MiB/s wr, 578 op/s 2024-04-22T15:15:19.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 118376 13.17 MB/sec execute 88 sec latency 24.741 ms 2024-04-22T15:15:20.023 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd='[{"prefix": "mgr fail", "who": "smithi186.vtjgwg"}]': finished 2024-04-22T15:15:20.023 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: mgrmap e29: smithi119.jjbxxd(active, starting, since 0.854998s) 2024-04-22T15:15:20.023 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:15:20.023 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:15:20.023 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:15:20.023 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi119.jjbxxd", "id": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:15:20.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: Manager daemon smithi119.jjbxxd is now available 2024-04-22T15:15:20.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:15:20.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:15:20.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:15:20.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:19 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:15:20.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.24421 ' entity='mgr.smithi186.vtjgwg' cmd='[{"prefix": "mgr fail", "who": "smithi186.vtjgwg"}]': finished 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: mgrmap e29: smithi119.jjbxxd(active, starting, since 0.854998s) 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi119.jjbxxd", "id": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:15:20.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:15:20.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:15:20.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:15:20.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:15:20.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:15:20.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: Manager daemon smithi119.jjbxxd is now available 2024-04-22T15:15:20.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:15:20.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:15:20.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:15:20.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:19 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:15:20.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 118860 13.19 MB/sec execute 89 sec latency 39.062 ms 2024-04-22T15:15:21.036 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:20 smithi119 ceph-mon[21835]: [22/Apr/2024:15:15:19] ENGINE Bus STARTING 2024-04-22T15:15:21.036 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:20 smithi119 ceph-mon[21835]: mgrmap e30: smithi119.jjbxxd(active, since 1.86745s) 2024-04-22T15:15:21.036 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:20 smithi119 ceph-mon[21835]: pgmap v3: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:15:21.036 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:20 smithi119 ceph-mon[21835]: [22/Apr/2024:15:15:20] ENGINE Serving on http://172.21.15.119:8765 2024-04-22T15:15:21.036 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:20 smithi119 ceph-mon[21835]: [22/Apr/2024:15:15:20] ENGINE Serving on https://172.21.15.119:7150 2024-04-22T15:15:21.036 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:20 smithi119 ceph-mon[21835]: [22/Apr/2024:15:15:20] ENGINE Bus STARTED 2024-04-22T15:15:21.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:20 smithi186 ceph-mon[24041]: [22/Apr/2024:15:15:19] ENGINE Bus STARTING 2024-04-22T15:15:21.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:20 smithi186 ceph-mon[24041]: mgrmap e30: smithi119.jjbxxd(active, since 1.86745s) 2024-04-22T15:15:21.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:20 smithi186 ceph-mon[24041]: pgmap v3: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:15:21.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:20 smithi186 ceph-mon[24041]: [22/Apr/2024:15:15:20] ENGINE Serving on http://172.21.15.119:8765 2024-04-22T15:15:21.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:20 smithi186 ceph-mon[24041]: [22/Apr/2024:15:15:20] ENGINE Serving on https://172.21.15.119:7150 2024-04-22T15:15:21.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:20 smithi186 ceph-mon[24041]: [22/Apr/2024:15:15:20] ENGINE Bus STARTED 2024-04-22T15:15:21.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 119487 13.19 MB/sec execute 90 sec latency 32.564 ms 2024-04-22T15:15:22.059 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:21 smithi119 ceph-mon[21835]: pgmap v4: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:15:22.059 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:21 smithi119 ceph-mon[21835]: mgrmap e31: smithi119.jjbxxd(active, since 2s) 2024-04-22T15:15:22.195 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:21 smithi186 ceph-mon[24041]: pgmap v4: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:15:22.195 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:21 smithi186 ceph-mon[24041]: mgrmap e31: smithi119.jjbxxd(active, since 2s) 2024-04-22T15:15:22.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 119946 13.17 MB/sec execute 91 sec latency 27.278 ms 2024-04-22T15:15:23.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 120391 13.21 MB/sec execute 92 sec latency 54.085 ms 2024-04-22T15:15:24.123 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:23 smithi119 ceph-mon[21835]: pgmap v5: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:15:24.123 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:23 smithi119 ceph-mon[21835]: mgrmap e32: smithi119.jjbxxd(active, since 4s) 2024-04-22T15:15:24.123 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:23 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:24.123 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:23 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:24.271 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:23 smithi186 ceph-mon[24041]: pgmap v5: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:15:24.271 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:23 smithi186 ceph-mon[24041]: mgrmap e32: smithi119.jjbxxd(active, since 4s) 2024-04-22T15:15:24.271 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:23 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:24.271 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:23 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:24.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 120734 13.21 MB/sec execute 93 sec latency 43.670 ms 2024-04-22T15:15:25.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 121027 13.19 MB/sec execute 94 sec latency 54.160 ms 2024-04-22T15:15:26.392 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:26 smithi119 ceph-mon[21835]: pgmap v6: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:15:26.392 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:26 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:26.392 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:26 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:26.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 121456 13.13 MB/sec execute 95 sec latency 47.798 ms 2024-04-22T15:15:26.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:26 smithi186 ceph-mon[24041]: pgmap v6: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-22T15:15:26.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:26 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:26.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:26 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:27.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 122218 13.21 MB/sec execute 96 sec latency 20.309 ms 2024-04-22T15:15:28.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 123025 13.24 MB/sec execute 97 sec latency 40.268 ms 2024-04-22T15:15:28.761 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:28 smithi119 ceph-mon[21835]: pgmap v7: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 4.7 MiB/s wr, 468 op/s 2024-04-22T15:15:28.761 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:28 smithi119 ceph-mon[21835]: Standby manager daemon smithi186.vtjgwg started 2024-04-22T15:15:28.761 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:28 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/1458789415' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/crt"}]: dispatch 2024-04-22T15:15:28.762 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:28 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/1458789415' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:15:28.762 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:28 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/1458789415' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/key"}]: dispatch 2024-04-22T15:15:28.762 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:28 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/1458789415' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:15:28.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:28 smithi186 ceph-mon[24041]: pgmap v7: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 4.7 MiB/s wr, 468 op/s 2024-04-22T15:15:28.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:28 smithi186 ceph-mon[24041]: Standby manager daemon smithi186.vtjgwg started 2024-04-22T15:15:28.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:28 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/1458789415' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/crt"}]: dispatch 2024-04-22T15:15:28.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:28 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/1458789415' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:15:28.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:28 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/1458789415' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/key"}]: dispatch 2024-04-22T15:15:28.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:28 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/1458789415' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:15:29.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 123504 13.21 MB/sec execute 98 sec latency 42.754 ms 2024-04-22T15:15:29.778 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:29 smithi119 ceph-mon[21835]: mgrmap e33: smithi119.jjbxxd(active, since 10s), standbys: smithi186.vtjgwg 2024-04-22T15:15:29.779 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:29 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi186.vtjgwg", "id": "smithi186.vtjgwg"}]: dispatch 2024-04-22T15:15:29.779 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:29 smithi119 ceph-mon[21835]: pgmap v8: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.7 MiB/s wr, 364 op/s 2024-04-22T15:15:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:29 smithi186 ceph-mon[24041]: mgrmap e33: smithi119.jjbxxd(active, since 10s), standbys: smithi186.vtjgwg 2024-04-22T15:15:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:29 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi186.vtjgwg", "id": "smithi186.vtjgwg"}]: dispatch 2024-04-22T15:15:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:29 smithi186 ceph-mon[24041]: pgmap v8: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.7 MiB/s wr, 364 op/s 2024-04-22T15:15:30.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 124006 13.26 MB/sec execute 99 sec latency 72.325 ms 2024-04-22T15:15:31.057 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:30 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:31.057 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:30 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:30 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:30 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:31.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 124429 13.28 MB/sec execute 100 sec latency 32.836 ms 2024-04-22T15:15:32.317 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:32 smithi119 ceph-mon[21835]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 6.7 MiB/s wr, 680 op/s 2024-04-22T15:15:32.318 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:32 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:32.318 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:32 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:32.459 INFO:teuthology.orchestra.run.smithi119.stdout: 5 124992 13.30 MB/sec execute 101 sec latency 39.177 ms 2024-04-22T15:15:32.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:32 smithi186 ceph-mon[24041]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 6.7 MiB/s wr, 680 op/s 2024-04-22T15:15:32.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:32 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:32.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:32 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:33.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 125424 13.22 MB/sec execute 102 sec latency 24.296 ms 2024-04-22T15:15:34.334 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:34 smithi119 ceph-mon[21835]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 6.1 MiB/s wr, 620 op/s 2024-04-22T15:15:34.334 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:34 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:15:34.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 125865 13.25 MB/sec execute 103 sec latency 50.235 ms 2024-04-22T15:15:34.577 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:34 smithi186 ceph-mon[24041]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 6.1 MiB/s wr, 620 op/s 2024-04-22T15:15:34.577 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:34 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:15:35.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 126342 13.23 MB/sec execute 104 sec latency 57.559 ms 2024-04-22T15:15:36.396 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:36 smithi119 ceph-mon[21835]: pgmap v11: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 6.1 MiB/s wr, 620 op/s 2024-04-22T15:15:36.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 126737 13.16 MB/sec execute 105 sec latency 61.574 ms 2024-04-22T15:15:36.639 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:15:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:36 smithi186 ceph-mon[24041]: pgmap v11: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 6.1 MiB/s wr, 620 op/s 2024-04-22T15:15:37.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 127229 13.18 MB/sec execute 106 sec latency 35.513 ms 2024-04-22T15:15:37.671 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:15:37.672 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (73s) 7s ago 10m 15.7M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:15:37.672 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (10m) 7s ago 10m 6405k - 17.2.0 e1d6a67b021e 37eb54a66662 2024-04-22T15:15:37.672 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (9m) 13s ago 9m 6405k - 17.2.0 e1d6a67b021e ee37991f5d20 2024-04-22T15:15:37.672 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (65s) 7s ago 9m 45.6M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:15:37.672 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (4m) 7s ago 4m 37.6M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:15:37.672 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (4m) 13s ago 4m 13.4M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:15:37.672 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (39s) 7s ago 11m 480M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:15:37.672 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283 running (2m) 13s ago 9m 82.6M - 19.0.0-3223-g5883d494 3d983c26e278 fb27c8a37962 2024-04-22T15:15:37.672 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (11m) 7s ago 11m 54.9M 2048M 17.2.0 e1d6a67b021e e31b6591d879 2024-04-22T15:15:37.672 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (9m) 13s ago 9m 45.4M 2048M 17.2.0 e1d6a67b021e a804e52c2c5e 2024-04-22T15:15:37.672 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (3m) 7s ago 3m 266M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:15:37.672 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (3m) 13s ago 3m 82.2M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:15:37.673 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (62s) 7s ago 9m 10.7M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:15:37.673 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (48s) 13s ago 9m 7964k - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:15:37.673 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (8m) 7s ago 8m 452M 1936M 17.2.0 e1d6a67b021e 9999e9684eba 2024-04-22T15:15:37.673 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (8m) 7s ago 8m 369M 1936M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:15:37.673 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (7m) 7s ago 7m 277M 1936M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:15:37.673 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (7m) 7s ago 7m 404M 1936M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:15:37.673 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (7m) 13s ago 7m 390M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:15:37.673 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (6m) 13s ago 6m 448M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:15:37.673 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (6m) 13s ago 6m 381M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:15:37.673 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (5m) 13s ago 5m 522M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:15:37.673 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (52s) 7s ago 9m 53.3M - 2.33.4 514e6a882f6e a9a9bd1ad50f 2024-04-22T15:15:38.164 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:38 smithi119 ceph-mon[21835]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:38.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:38 smithi119 ceph-mon[21835]: pgmap v12: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 9.0 MiB/s wr, 904 op/s 2024-04-22T15:15:38.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:38 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:38.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:38 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:38.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:38 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:15:38.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:38 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:15:38.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:38 smithi119 ceph-mon[21835]: Updating smithi119:/etc/ceph/ceph.conf 2024-04-22T15:15:38.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:38 smithi119 ceph-mon[21835]: Updating smithi186:/etc/ceph/ceph.conf 2024-04-22T15:15:38.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:38 smithi119 ceph-mon[21835]: from='client.14800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:38.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:38 smithi119 ceph-mon[21835]: Updating smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:15:38.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:38 smithi119 ceph-mon[21835]: Updating smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:15:38.368 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:15:38.368 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:15:38.368 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:15:38.368 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:15:38.368 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:15:38.368 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:15:38.369 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:15:38.369 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:15:38.369 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:15:38.369 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:15:38.369 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:15:38.369 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:15:38.369 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:15:38.369 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:15:38.369 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-22T15:15:38.369 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:15:38.369 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:15:38.369 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:15:38.452 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:38 smithi186 ceph-mon[24041]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:38.453 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:38 smithi186 ceph-mon[24041]: pgmap v12: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 9.0 MiB/s wr, 904 op/s 2024-04-22T15:15:38.453 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:38 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:38.453 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:38 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:38.453 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:38 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:15:38.453 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:38 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:15:38.453 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:38 smithi186 ceph-mon[24041]: Updating smithi119:/etc/ceph/ceph.conf 2024-04-22T15:15:38.453 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:38 smithi186 ceph-mon[24041]: Updating smithi186:/etc/ceph/ceph.conf 2024-04-22T15:15:38.453 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:38 smithi186 ceph-mon[24041]: from='client.14800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:38.453 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:38 smithi186 ceph-mon[24041]: Updating smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:15:38.453 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:38 smithi186 ceph-mon[24041]: Updating smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:15:38.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 127861 13.28 MB/sec execute 107 sec latency 32.283 ms 2024-04-22T15:15:38.919 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:15:38.919 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:15:38.919 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:15:38.919 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:15:38.919 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:15:38.919 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr" 2024-04-22T15:15:38.919 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:15:38.919 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "2/23 daemons upgraded", 2024-04-22T15:15:38.919 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "", 2024-04-22T15:15:38.919 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:15:38.919 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:15:39.171 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:39 smithi119 ceph-mon[21835]: from='client.14804 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:39.171 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:39 smithi119 ceph-mon[21835]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:15:39.172 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:39 smithi119 ceph-mon[21835]: Updating smithi119:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:15:39.172 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:39 smithi119 ceph-mon[21835]: Updating smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:15:39.172 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:39 smithi119 ceph-mon[21835]: Updating smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:15:39.172 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:39 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/2747664304' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:15:39.172 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:39 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:39.172 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:39 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:39.172 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:39 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:39.172 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:39 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:39.172 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:39 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:39.172 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:39 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:39.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 128261 13.30 MB/sec execute 108 sec latency 43.819 ms 2024-04-22T15:15:39.472 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:15:39.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:39 smithi186 ceph-mon[24041]: from='client.14804 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:39.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:39 smithi186 ceph-mon[24041]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:15:39.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:39 smithi186 ceph-mon[24041]: Updating smithi119:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:15:39.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:39 smithi186 ceph-mon[24041]: Updating smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:15:39.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:39 smithi186 ceph-mon[24041]: Updating smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:15:39.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:39 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/2747664304' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:15:39.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:39 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:39.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:39 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:39.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:39 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:39.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:39 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:39.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:39 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:39.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:39 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:40.188 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:40 smithi119 ceph-mon[21835]: Reconfiguring prometheus.smithi119 (dependencies changed)... 2024-04-22T15:15:40.188 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:40 smithi119 ceph-mon[21835]: Reconfiguring daemon prometheus.smithi119 on smithi119 2024-04-22T15:15:40.188 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:40 smithi119 ceph-mon[21835]: from='client.14810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:40.188 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:40 smithi119 ceph-mon[21835]: pgmap v13: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.3 MiB/s wr, 632 op/s 2024-04-22T15:15:40.188 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:40 smithi119 ceph-mon[21835]: from='client.? 172.21.15.119:0/1569058480' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:15:40.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 128568 13.20 MB/sec execute 109 sec latency 48.920 ms 2024-04-22T15:15:40.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:40 smithi186 ceph-mon[24041]: Reconfiguring prometheus.smithi119 (dependencies changed)... 2024-04-22T15:15:40.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:40 smithi186 ceph-mon[24041]: Reconfiguring daemon prometheus.smithi119 on smithi119 2024-04-22T15:15:40.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:40 smithi186 ceph-mon[24041]: from='client.14810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:40.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:40 smithi186 ceph-mon[24041]: pgmap v13: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.3 MiB/s wr, 632 op/s 2024-04-22T15:15:40.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:40 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/1569058480' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:15:41.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 128848 13.11 MB/sec execute 110 sec latency 37.260 ms 2024-04-22T15:15:42.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 129148 13.07 MB/sec execute 111 sec latency 48.343 ms 2024-04-22T15:15:42.749 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:42 smithi119 ceph-mon[21835]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 904 op/s 2024-04-22T15:15:43.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:42 smithi186 ceph-mon[24041]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 904 op/s 2024-04-22T15:15:43.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 129457 13.05 MB/sec execute 112 sec latency 42.287 ms 2024-04-22T15:15:44.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 129869 13.03 MB/sec execute 113 sec latency 58.039 ms 2024-04-22T15:15:44.767 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:44 smithi119 ceph-mon[21835]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 555 op/s 2024-04-22T15:15:45.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:44 smithi186 ceph-mon[24041]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 555 op/s 2024-04-22T15:15:45.460 INFO:teuthology.orchestra.run.smithi119.stdout: 5 130254 12.95 MB/sec execute 114 sec latency 42.235 ms 2024-04-22T15:15:46.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 130574 12.92 MB/sec execute 115 sec latency 38.552 ms 2024-04-22T15:15:46.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:46 smithi186 ceph-mon[24041]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 555 op/s 2024-04-22T15:15:46.798 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:46 smithi119 ceph-mon[21835]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 555 op/s 2024-04-22T15:15:47.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 130919 12.91 MB/sec execute 116 sec latency 47.198 ms 2024-04-22T15:15:47.828 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:47 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:47.829 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:47 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:47.829 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:47 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:15:47.829 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:47 smithi119 ceph-mon[21835]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:15:47.829 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:47 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi119:9095"}]: dispatch 2024-04-22T15:15:47.829 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:47 smithi119 ceph-mon[21835]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi119:9095"}]: dispatch 2024-04-22T15:15:47.829 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:47 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:47.829 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:47 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:15:47.829 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:47 smithi119 ceph-mon[21835]: pgmap v17: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 765 op/s 2024-04-22T15:15:47.892 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:47 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:47.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:47 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:47.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:47 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:15:47.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:47 smithi186 ceph-mon[24041]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:15:47.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:47 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi119:9095"}]: dispatch 2024-04-22T15:15:47.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:47 smithi186 ceph-mon[24041]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi119:9095"}]: dispatch 2024-04-22T15:15:47.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:47 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:47.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:47 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:15:47.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:47 smithi186 ceph-mon[24041]: pgmap v17: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 765 op/s 2024-04-22T15:15:48.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 131285 12.92 MB/sec execute 117 sec latency 55.985 ms 2024-04-22T15:15:49.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 131756 12.97 MB/sec execute 118 sec latency 34.390 ms 2024-04-22T15:15:50.092 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:50 smithi119 ceph-mon[21835]: pgmap v18: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.7 MiB/s wr, 481 op/s 2024-04-22T15:15:50.092 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:50 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:15:50.092 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:50 smithi119 ceph-mon[21835]: Upgrade: Updating mgr.smithi186.vtjgwg 2024-04-22T15:15:50.093 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:50 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi186.vtjgwg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:15:50.093 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:50 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:15:50.093 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:50 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:15:50.093 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:50 smithi119 ceph-mon[21835]: Deploying daemon mgr.smithi186.vtjgwg on smithi186 2024-04-22T15:15:50.169 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:50 smithi186 ceph-mon[24041]: pgmap v18: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.7 MiB/s wr, 481 op/s 2024-04-22T15:15:50.170 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:50 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:15:50.170 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:50 smithi186 ceph-mon[24041]: Upgrade: Updating mgr.smithi186.vtjgwg 2024-04-22T15:15:50.170 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:50 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi186.vtjgwg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:15:50.170 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:50 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:15:50.170 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:50 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:15:50.170 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:50 smithi186 ceph-mon[24041]: Deploying daemon mgr.smithi186.vtjgwg on smithi186 2024-04-22T15:15:50.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 132205 12.91 MB/sec execute 119 sec latency 33.376 ms 2024-04-22T15:15:51.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 132458 12.83 MB/sec execute 120 sec latency 49.493 ms 2024-04-22T15:15:52.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 132751 12.81 MB/sec execute 121 sec latency 45.052 ms 2024-04-22T15:15:52.661 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:52 smithi119 ceph-mon[21835]: pgmap v19: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 714 op/s 2024-04-22T15:15:53.015 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:52 smithi186 ceph-mon[24041]: pgmap v19: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 714 op/s 2024-04-22T15:15:53.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 133016 12.76 MB/sec execute 122 sec latency 62.401 ms 2024-04-22T15:15:54.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 133327 12.72 MB/sec execute 123 sec latency 74.647 ms 2024-04-22T15:15:54.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:54 smithi186 ceph-mon[24041]: pgmap v20: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 442 op/s 2024-04-22T15:15:54.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:54 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:54.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:54 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:54.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:54 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:15:54.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:54 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:54.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:54 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:54.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:54 smithi119 ceph-mon[21835]: pgmap v20: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 442 op/s 2024-04-22T15:15:54.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:54 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:54.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:54 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:54.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:54 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:15:54.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:54 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:54.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:54 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:55.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 133698 12.67 MB/sec execute 124 sec latency 88.869 ms 2024-04-22T15:15:56.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 134049 12.63 MB/sec execute 125 sec latency 41.884 ms 2024-04-22T15:15:56.707 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:56 smithi119 ceph-mon[21835]: pgmap v21: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 442 op/s 2024-04-22T15:15:56.760 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:56 smithi186 ceph-mon[24041]: pgmap v21: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 442 op/s 2024-04-22T15:15:57.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 134429 12.64 MB/sec execute 126 sec latency 38.731 ms 2024-04-22T15:15:58.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 134841 12.66 MB/sec execute 127 sec latency 40.377 ms 2024-04-22T15:15:58.652 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:58 smithi186 ceph-mon[24041]: pgmap v22: 97 pgs: 97 active+clean; 323 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 638 op/s 2024-04-22T15:15:58.751 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:58 smithi119 ceph-mon[21835]: pgmap v22: 97 pgs: 97 active+clean; 323 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 638 op/s 2024-04-22T15:15:59.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 135271 12.71 MB/sec execute 128 sec latency 41.878 ms 2024-04-22T15:15:59.760 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:59 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:59.760 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:59 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:15:59.761 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:15:59 smithi119 ceph-mon[21835]: pgmap v23: 97 pgs: 97 active+clean; 323 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 428 op/s 2024-04-22T15:16:00.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:59 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:00.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:59 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:00.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:15:59 smithi186 ceph-mon[24041]: pgmap v23: 97 pgs: 97 active+clean; 323 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 428 op/s 2024-04-22T15:16:00.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 135584 12.63 MB/sec execute 129 sec latency 47.286 ms 2024-04-22T15:16:01.152 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:01 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:01.152 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:01 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:01.315 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:01 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:01.315 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:01 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:01.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 135826 12.55 MB/sec execute 130 sec latency 86.943 ms 2024-04-22T15:16:02.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 136002 12.48 MB/sec execute 131 sec latency 70.764 ms 2024-04-22T15:16:02.855 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:02 smithi186 ceph-mon[24041]: pgmap v24: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.9 MiB/s wr, 655 op/s 2024-04-22T15:16:02.855 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:02 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:02.855 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:02 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:02.857 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:02 smithi119 ceph-mon[21835]: pgmap v24: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.9 MiB/s wr, 655 op/s 2024-04-22T15:16:02.857 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:02 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:02.857 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:02 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:03.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 136257 12.46 MB/sec execute 132 sec latency 46.335 ms 2024-04-22T15:16:03.645 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:03 smithi119 ceph-mon[21835]: Standby manager daemon smithi186.vtjgwg restarted 2024-04-22T15:16:03.645 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:03 smithi119 ceph-mon[21835]: Standby manager daemon smithi186.vtjgwg started 2024-04-22T15:16:03.645 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:03 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/1766131117' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/crt"}]: dispatch 2024-04-22T15:16:03.646 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:03 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/1766131117' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:16:03.646 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:03 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/1766131117' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/key"}]: dispatch 2024-04-22T15:16:03.646 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:03 smithi119 ceph-mon[21835]: from='mgr.? 172.21.15.186:0/1766131117' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:16:03.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:03 smithi186 ceph-mon[24041]: Standby manager daemon smithi186.vtjgwg restarted 2024-04-22T15:16:03.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:03 smithi186 ceph-mon[24041]: Standby manager daemon smithi186.vtjgwg started 2024-04-22T15:16:03.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:03 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/1766131117' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/crt"}]: dispatch 2024-04-22T15:16:03.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:03 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/1766131117' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:16:03.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:03 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/1766131117' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/key"}]: dispatch 2024-04-22T15:16:03.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:03 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/1766131117' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:16:04.462 INFO:teuthology.orchestra.run.smithi119.stdout: 5 136492 12.39 MB/sec execute 133 sec latency 100.100 ms 2024-04-22T15:16:04.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: pgmap v25: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-04-22T15:16:04.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: mgrmap e34: smithi119.jjbxxd(active, since 45s), standbys: smithi186.vtjgwg 2024-04-22T15:16:04.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:16:04.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:04.658 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:04.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:04.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:16:04.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:04.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:04.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:16:04.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:16:04.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:04.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi119.jjbxxd"}]: dispatch 2024-04-22T15:16:04.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi119.jjbxxd"}]': finished 2024-04-22T15:16:04.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi186.vtjgwg"}]: dispatch 2024-04-22T15:16:04.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi186.vtjgwg"}]': finished 2024-04-22T15:16:04.659 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:04 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:16:05.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: pgmap v25: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-04-22T15:16:05.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: mgrmap e34: smithi119.jjbxxd(active, since 45s), standbys: smithi186.vtjgwg 2024-04-22T15:16:05.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:16:05.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:05.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:05.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:05.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:16:05.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:05.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:05.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:16:05.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:16:05.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:05.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi119.jjbxxd"}]: dispatch 2024-04-22T15:16:05.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi119.jjbxxd"}]': finished 2024-04-22T15:16:05.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi186.vtjgwg"}]: dispatch 2024-04-22T15:16:05.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi186.vtjgwg"}]': finished 2024-04-22T15:16:05.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:04 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:16:05.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 136989 12.42 MB/sec execute 134 sec latency 46.218 ms 2024-04-22T15:16:05.667 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:05 smithi119 ceph-mon[21835]: Upgrade: Setting container_image for all mgr 2024-04-22T15:16:06.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:05 smithi186 ceph-mon[24041]: Upgrade: Setting container_image for all mgr 2024-04-22T15:16:06.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 137475 12.38 MB/sec execute 135 sec latency 39.425 ms 2024-04-22T15:16:06.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:06 smithi119 ceph-mon[21835]: pgmap v26: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-04-22T15:16:07.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:06 smithi186 ceph-mon[24041]: pgmap v26: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-04-22T15:16:07.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 137918 12.38 MB/sec execute 136 sec latency 29.997 ms 2024-04-22T15:16:08.226 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:07 smithi119 ceph-mon[21835]: Upgrade: Updating mon.smithi119 2024-04-22T15:16:08.226 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:07 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:08.226 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:07 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:16:08.227 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:07 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:16:08.227 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:07 smithi119 ceph-mon[21835]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:08.227 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:07 smithi119 ceph-mon[21835]: Deploying daemon mon.smithi119 on smithi119 2024-04-22T15:16:08.227 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:07 smithi119 ceph-mon[21835]: pgmap v27: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.8 MiB/s wr, 594 op/s 2024-04-22T15:16:08.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:07 smithi186 ceph-mon[24041]: Upgrade: Updating mon.smithi119 2024-04-22T15:16:08.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:07 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:08.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:07 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:16:08.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:07 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:16:08.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:07 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:08.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:07 smithi186 ceph-mon[24041]: Deploying daemon mon.smithi119 on smithi119 2024-04-22T15:16:08.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:07 smithi186 ceph-mon[24041]: pgmap v27: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.8 MiB/s wr, 594 op/s 2024-04-22T15:16:08.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 138391 12.44 MB/sec execute 137 sec latency 31.719 ms 2024-04-22T15:16:09.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 138831 12.47 MB/sec execute 138 sec latency 44.702 ms 2024-04-22T15:16:10.015 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:16:10.037 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:10 smithi119 systemd[1]: Stopping Ceph mon.smithi119 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:16:10.298 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:10 smithi119 ceph-mon[21835]: pgmap v28: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 398 op/s 2024-04-22T15:16:10.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 139110 12.41 MB/sec execute 139 sec latency 77.663 ms 2024-04-22T15:16:10.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:10 smithi186 ceph-mon[24041]: pgmap v28: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 398 op/s 2024-04-22T15:16:10.558 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:10 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119[21831]: 2024-04-22T15:16:10.515+0000 7f4bfeceb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi119 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:16:10.558 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:10 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119[21831]: 2024-04-22T15:16:10.515+0000 7f4bfeceb700 -1 mon.smithi119@0(leader) e2 *** Got Signal Terminated *** 2024-04-22T15:16:10.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:10 smithi119 podman[62900]: 2024-04-22 15:16:10.56163508 +0000 UTC m=+0.306017454 container died e31b6591d879b56cac373b321e58fad7bc689660f99a38872f480090f3e06685 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, ceph=True, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, maintainer=Guillaume Abrioux , release=754, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, RELEASE=HEAD, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream) 2024-04-22T15:16:10.981 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:16:10.982 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (107s) 9s ago 10m 15.7M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:16:10.982 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (10m) 9s ago 10m 6405k - 17.2.0 e1d6a67b021e 37eb54a66662 2024-04-22T15:16:10.982 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (10m) 12s ago 10m 6405k - 17.2.0 e1d6a67b021e ee37991f5d20 2024-04-22T15:16:10.982 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (98s) 9s ago 10m 44.1M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:16:10.982 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (4m) 9s ago 4m 38.1M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:16:10.982 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (4m) 12s ago 4m 13.6M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:16:10.982 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (72s) 9s ago 12m 497M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:16:10.982 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (18s) 12s ago 10m 105M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:16:10.982 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (12m) 9s ago 12m 56.7M 2048M 17.2.0 e1d6a67b021e e31b6591d879 2024-04-22T15:16:10.982 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (10m) 12s ago 10m 45.7M 2048M 17.2.0 e1d6a67b021e a804e52c2c5e 2024-04-22T15:16:10.982 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (4m) 9s ago 4m 268M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:16:10.982 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (4m) 12s ago 4m 82.3M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:16:10.983 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (95s) 9s ago 10m 11.0M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:16:10.983 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (82s) 12s ago 9m 8610k - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:16:10.983 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (9m) 9s ago 9m 479M 1936M 17.2.0 e1d6a67b021e 9999e9684eba 2024-04-22T15:16:10.983 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (8m) 9s ago 8m 410M 1936M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:16:10.983 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (8m) 9s ago 8m 291M 1936M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:16:10.983 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (7m) 9s ago 7m 433M 1936M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:16:10.983 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (7m) 12s ago 7m 458M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:16:10.983 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (7m) 12s ago 7m 507M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:16:10.983 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (6m) 12s ago 6m 422M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:16:10.983 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (6m) 12s ago 6m 326M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:16:10.983 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (24s) 9s ago 9m 53.5M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:16:11.461 INFO:teuthology.orchestra.run.smithi119.stdout: 5 139340 12.34 MB/sec execute 140 sec latency 124.843 ms 2024-04-22T15:16:11.605 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:16:11.605 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:16:11.605 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:16:11.605 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:16:11.605 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:16:11.605 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:16:11.605 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:16:11.605 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:16:11.605 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:16:11.605 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:16:11.606 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:16:11.606 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:16:11.606 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:16:11.606 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:16:11.606 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-22T15:16:11.606 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:16:11.606 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:16:11.606 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:16:11.623 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:11 smithi119 podman[62900]: 2024-04-22 15:16:11.600330693 +0000 UTC m=+1.344713061 container cleanup e31b6591d879b56cac373b321e58fad7bc689660f99a38872f480090f3e06685 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, name=centos-stream, release=754, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0) 2024-04-22T15:16:11.623 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:11 smithi119 bash[62900]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119 2024-04-22T15:16:11.880 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:11 smithi119 podman[62914]: 2024-04-22 15:16:11.863836397 +0000 UTC m=+1.298463331 container remove e31b6591d879b56cac373b321e58fad7bc689660f99a38872f480090f3e06685 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vendor=Red Hat, Inc., release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.component=centos-stream-container, name=centos-stream, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8) 2024-04-22T15:16:12.137 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:16:12.137 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:16:12.137 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:16:12.137 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:16:12.137 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:16:12.137 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr" 2024-04-22T15:16:12.137 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:16:12.137 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "2/23 daemons upgraded", 2024-04-22T15:16:12.137 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading mon daemons", 2024-04-22T15:16:12.137 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:16:12.137 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:16:12.383 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:12 smithi119 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi119.service: Deactivated successfully. 2024-04-22T15:16:12.383 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:12 smithi119 systemd[1]: Stopped Ceph mon.smithi119 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:16:12.384 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:12 smithi119 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi119.service: Consumed 15.845s CPU time. 2024-04-22T15:16:12.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 139591 12.28 MB/sec execute 141 sec latency 61.010 ms 2024-04-22T15:16:12.827 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:16:12.907 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:12 smithi119 systemd[1]: Starting Ceph mon.smithi119 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:16:13.158 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:12 smithi119 podman[63136]: 2024-04-22 15:16:12.838994304 +0000 UTC m=+0.029629251 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:16:13.159 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 podman[63136]: 2024-04-22 15:16:13.012193635 +0000 UTC m=+0.202828565 container create 6f921c4637d27c079d3cc73d868bbb8a405800f988e04b8b4c34785f12447ffe (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-22T15:16:13.418 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 podman[63136]: 2024-04-22 15:16:13.378249101 +0000 UTC m=+0.568884040 container init 6f921c4637d27c079d3cc73d868bbb8a405800f988e04b8b4c34785f12447ffe (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:16:13.419 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 podman[63136]: 2024-04-22 15:16:13.38515012 +0000 UTC m=+0.575785053 container start 6f921c4637d27c079d3cc73d868bbb8a405800f988e04b8b4c34785f12447ffe (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-22T15:16:13.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 139892 12.26 MB/sec execute 142 sec latency 57.845 ms 2024-04-22T15:16:13.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T15:16:13.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev), process ceph-mon, pid 2 2024-04-22T15:16:13.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: pidfile_write: ignore empty --pid-file 2024-04-22T15:16:13.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: load: jerasure load: lrc 2024-04-22T15:16:13.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: RocksDB version: 7.9.2 2024-04-22T15:16:13.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Git sha 0 2024-04-22T15:16:13.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Compile date 2024-04-22 12:55:29 2024-04-22T15:16:13.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: DB SUMMARY 2024-04-22T15:16:13.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: DB Session ID: 97WPKTXP4GD5TFY3U9MF 2024-04-22T15:16:13.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: CURRENT file: CURRENT 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: MANIFEST file: MANIFEST-000015 size: 979 Bytes 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi119/store.db dir, Total Num: 1, files: 000030.sst 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi119/store.db: 000028.log size: 1571122 ; 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.error_if_exists: 0 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.create_if_missing: 0 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.paranoid_checks: 1 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.env: 0x557e39dcdcc0 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.fs: PosixFileSystem 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.info_log: 0x557e3b837aa0 2024-04-22T15:16:13.671 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.statistics: (nil) 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.use_fsync: 0 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_log_file_size: 0 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.allow_fallocate: 1 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.use_direct_reads: 0 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T15:16:13.672 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.db_log_dir: 2024-04-22T15:16:13.673 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.wal_dir: 2024-04-22T15:16:13.673 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T15:16:13.673 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T15:16:13.673 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T15:16:13.673 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.write_buffer_manager: 0x557e3b7cb0e0 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T15:16:13.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.unordered_write: 0 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.row_cache: None 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.wal_filter: None 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.two_write_queues: 0 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.wal_compression: 0 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.atomic_flush: 0 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T15:16:13.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.log_readahead_size: 0 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_background_jobs: 2 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_background_compactions: -1 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_subcompactions: 1 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T15:16:13.676 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_open_files: -1 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_background_flushes: -1 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Compression algorithms supported: 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T15:16:13.677 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: kZSTD supported: 0 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: kXpressCompression supported: 0 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: kZlibCompression supported: 1 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: kSnappyCompression supported: 1 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: kLZ4Compression supported: 1 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: kBZip2Compression supported: 0 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi119/store.db/MANIFEST-000015 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.merge_operator: 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_filter: None 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T15:16:13.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557e3b837c80) 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: cache_index_and_filter_blocks: 1 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: pin_top_level_index_and_filter: 1 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: index_type: 0 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: data_block_index_type: 0 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: index_shortening: 1 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: checksum: 4 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: no_block_cache: 0 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_cache: 0x557e3b7fd1f0 2024-04-22T15:16:13.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_cache_name: BinnedLRUCache 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_cache_options: 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: capacity : 536870912 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: num_shard_bits : 4 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: strict_capacity_limit : 0 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: high_pri_pool_ratio: 0.000 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_cache_compressed: (nil) 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: persistent_cache: (nil) 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_size: 4096 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_size_deviation: 10 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_restart_interval: 16 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: index_block_restart_interval: 1 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: metadata_block_size: 4096 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: partition_filters: 0 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: use_delta_encoding: 1 2024-04-22T15:16:13.680 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: filter_policy: bloomfilter 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: whole_key_filtering: 1 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: verify_compression: 0 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: read_amp_bytes_per_bit: 0 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: format_version: 5 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: enable_index_compression: 1 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: block_align: 0 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: max_auto_readahead_size: 262144 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: prepopulate_block_cache: 0 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: initial_auto_readahead_size: 8192 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: num_file_reads_for_auto_readahead: 2 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compression: NoCompression 2024-04-22T15:16:13.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T15:16:13.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T15:16:13.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T15:16:13.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.num_levels: 7 2024-04-22T15:16:13.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T15:16:13.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T15:16:13.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T15:16:13.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T15:16:13.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T15:16:13.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T15:16:13.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T15:16:13.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:16:13.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T15:16:13.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T15:16:13.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-22T15:16:13.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-22T15:16:13.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T15:16:13.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T15:16:13.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T15:16:13.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T15:16:13.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:16:13.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T15:16:13.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.arena_block_size: 1048576 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T15:16:13.685 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.inplace_update_support: 0 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.bloom_locality: 0 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.max_successive_merges: 0 2024-04-22T15:16:13.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.ttl: 2592000 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.enable_blob_files: false 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.min_blob_size: 0 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-22T15:16:13.687 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.blob_file_starting_level: 0 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi119/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11864, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 8f6587a7-0272-4f02-ac4a-ea1b6873a482 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798973423287, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798973429172, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 1394820, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11865, "largest_seqno": 12678, "table_properties": {"data_size": 1390421, "index_size": 2463, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 901, "raw_key_size": 8221, "raw_average_key_size": 23, "raw_value_size": 1382587, "raw_average_value_size": 3995, "num_data_blocks": 112, "num_entries": 346, "num_filter_entries": 346, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1713798973, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "8f6587a7-0272-4f02-ac4a-ea1b6873a482", "db_session_id": "97WPKTXP4GD5TFY3U9MF", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798973429313, "job": 1, "event": "recovery_finished"} 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T15:16:13.688 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi119/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x557e3b912000 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: DB pointer 0x557e3b8fc000 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ** DB Stats ** 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:16:13.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ** Compaction Stats [default] ** 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: L0 1/0 1.33 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 424.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: L6 1/0 7.38 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Sum 2/0 8.71 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 424.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 424.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ** Compaction Stats [default] ** 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 424.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:16:13.690 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Cumulative compaction: 0.00 GB write, 153.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Interval compaction: 0.00 GB write, 153.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Block cache BinnedLRUCache@0x557e3b7fd1f0#2 capacity: 512.00 MB usage: 3.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,2.58 KB,0.000491738%) Misc(1,0.00 KB,0%) 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:16:13.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: starting mon.smithi119 rank 0 at public addrs [v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0] at bind addrs [v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi119 fsid 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: mon.smithi119@-1(???) e2 preinit fsid 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: mon.smithi119@-1(???).mds e7 new map 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: mon.smithi119@-1(???).mds e7 print_map 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: e7 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: legacy client fscid: 1 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Filesystem 'foofs' (1) 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: fs_name foofs 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: epoch 5 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-22T15:16:13.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: created 2024-04-22T15:11:14.746120+0000 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: modified 2024-04-22T15:11:20.769028+0000 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: tableserver 0 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: root 0 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: session_timeout 60 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: session_autoclose 300 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: max_file_size 1099511627776 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: max_xattr_size 65536 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: required_client_features {} 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: last_failure 0 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: last_failure_osd_epoch 0 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: max_mds 1 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: in 0 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: up {0=14490} 2024-04-22T15:16:13.693 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: failed 2024-04-22T15:16:13.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: damaged 2024-04-22T15:16:13.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: stopped 2024-04-22T15:16:13.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: data_pools [3] 2024-04-22T15:16:13.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: metadata_pool 2 2024-04-22T15:16:13.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: inline_data disabled 2024-04-22T15:16:13.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: balancer 2024-04-22T15:16:13.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: bal_rank_mask -1 2024-04-22T15:16:13.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: standby_count_wanted 1 2024-04-22T15:16:13.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: qdb_cluster leader: 0 members: 2024-04-22T15:16:13.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: [mds.foofs.smithi119.ncbqaf{0:14490} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.119:6834/3732854443,v1:172.21.15.119:6835/3732854443] compat {c=[1],r=[1],i=[7ff]}] 2024-04-22T15:16:13.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:16:13.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:16:13.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: Standby daemons: 2024-04-22T15:16:13.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: 2024-04-22T15:16:13.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout: [mds.foofs.smithi186.dglszf{-1:24281} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.186:6832/2825961187,v1:172.21.15.186:6833/2825961187] compat {c=[1],r=[1],i=[7ff]}] 2024-04-22T15:16:13.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: mon.smithi119@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-22T15:16:13.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: mon.smithi119@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:16:13.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: mon.smithi119@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:16:13.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: mon.smithi119@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:16:13.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: mon.smithi119@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-04-22T15:16:13.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 bash[63136]: 6f921c4637d27c079d3cc73d868bbb8a405800f988e04b8b4c34785f12447ffe 2024-04-22T15:16:13.695 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 systemd[1]: Started Ceph mon.smithi119 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:16:13.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: from='client.24571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:16:13.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:16:13.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:16:13.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: from='client.24575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:16:13.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: pgmap v29: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.0 MiB/s wr, 661 op/s 2024-04-22T15:16:13.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/3051138472' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:16:13.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: from='client.24583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:16:13.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/3205246966' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:16:13.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: pgmap v30: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 434 op/s 2024-04-22T15:16:13.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: mon.smithi119 calling monitor election 2024-04-22T15:16:13.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: mon.smithi119 is new leader, mons smithi119,smithi186 in quorum (ranks 0,1) 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: monmap epoch 2 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: fsid 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: last_changed 2024-04-22T15:06:06.798166+0000 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: created 2024-04-22T15:03:50.504762+0000 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: min_mon_release 17 (quincy) 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: election_strategy: 1 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: 0: [v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0] mon.smithi119 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: 1: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.smithi186 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 1 up:standby 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: mgrmap e34: smithi119.jjbxxd(active, since 55s), standbys: smithi186.vtjgwg 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: overall HEALTH_OK 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: from='mgr.14744 ' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:13.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: mgrmap e35: smithi119.jjbxxd(active, since 55s), standbys: smithi186.vtjgwg 2024-04-22T15:16:13.923 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:13 smithi119 ceph-mon[63156]: from='mgr.14744 ' entity='' 2024-04-22T15:16:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: from='client.24571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:16:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:16:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: from='mgr.14744 172.21.15.119:0/3272917330' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:16:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: from='client.24575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:16:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: pgmap v29: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.0 MiB/s wr, 661 op/s 2024-04-22T15:16:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/3051138472' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:16:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: from='client.24583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:16:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: from='client.? 172.21.15.119:0/3205246966' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:16:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: pgmap v30: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 434 op/s 2024-04-22T15:16:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: mon.smithi119 calling monitor election 2024-04-22T15:16:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: mon.smithi119 is new leader, mons smithi119,smithi186 in quorum (ranks 0,1) 2024-04-22T15:16:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: monmap epoch 2 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: fsid 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: last_changed 2024-04-22T15:06:06.798166+0000 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: created 2024-04-22T15:03:50.504762+0000 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: min_mon_release 17 (quincy) 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: election_strategy: 1 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: 0: [v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0] mon.smithi119 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: 1: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.smithi186 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 1 up:standby 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: mgrmap e34: smithi119.jjbxxd(active, since 55s), standbys: smithi186.vtjgwg 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: overall HEALTH_OK 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: from='mgr.14744 ' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: mgrmap e35: smithi119.jjbxxd(active, since 55s), standbys: smithi186.vtjgwg 2024-04-22T15:16:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:13 smithi186 ceph-mon[24041]: from='mgr.14744 ' entity='' 2024-04-22T15:16:14.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 140512 12.31 MB/sec execute 143 sec latency 35.995 ms 2024-04-22T15:16:15.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 141142 12.29 MB/sec execute 144 sec latency 25.224 ms 2024-04-22T15:16:15.679 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:15 smithi119 ceph-mon[63156]: mgrmap e36: smithi119.jjbxxd(active, since 56s), standbys: smithi186.vtjgwg 2024-04-22T15:16:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:15 smithi186 ceph-mon[24041]: mgrmap e36: smithi119.jjbxxd(active, since 56s), standbys: smithi186.vtjgwg 2024-04-22T15:16:16.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 141850 12.39 MB/sec execute 145 sec latency 23.702 ms 2024-04-22T15:16:17.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 142561 12.46 MB/sec execute 146 sec latency 15.267 ms 2024-04-22T15:16:18.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 143039 12.42 MB/sec execute 147 sec latency 61.923 ms 2024-04-22T15:16:19.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 143643 12.44 MB/sec execute 148 sec latency 25.378 ms 2024-04-22T15:16:20.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 144356 12.47 MB/sec execute 149 sec latency 24.775 ms 2024-04-22T15:16:21.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 144987 12.50 MB/sec execute 150 sec latency 36.488 ms 2024-04-22T15:16:22.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 145752 12.63 MB/sec execute 151 sec latency 25.532 ms 2024-04-22T15:16:22.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:22 smithi119 ceph-mon[63156]: Standby manager daemon smithi186.vtjgwg restarted 2024-04-22T15:16:22.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:22 smithi119 ceph-mon[63156]: Standby manager daemon smithi186.vtjgwg started 2024-04-22T15:16:22.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:22 smithi119 ceph-mon[63156]: from='mgr.? 172.21.15.186:0/496737871' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/crt"}]: dispatch 2024-04-22T15:16:22.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:22 smithi119 ceph-mon[63156]: from='mgr.? 172.21.15.186:0/496737871' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:16:22.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:22 smithi119 ceph-mon[63156]: from='mgr.? 172.21.15.186:0/496737871' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/key"}]: dispatch 2024-04-22T15:16:22.675 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:22 smithi119 ceph-mon[63156]: from='mgr.? 172.21.15.186:0/496737871' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:16:22.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:22 smithi186 ceph-mon[24041]: Standby manager daemon smithi186.vtjgwg restarted 2024-04-22T15:16:22.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:22 smithi186 ceph-mon[24041]: Standby manager daemon smithi186.vtjgwg started 2024-04-22T15:16:22.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:22 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/496737871' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/crt"}]: dispatch 2024-04-22T15:16:22.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:22 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/496737871' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:16:22.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:22 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/496737871' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi186.vtjgwg/key"}]: dispatch 2024-04-22T15:16:22.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:22 smithi186 ceph-mon[24041]: from='mgr.? 172.21.15.186:0/496737871' entity='mgr.smithi186.vtjgwg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:16:23.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 146233 12.60 MB/sec execute 152 sec latency 46.273 ms 2024-04-22T15:16:23.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:23 smithi119 ceph-mon[63156]: mgrmap e37: smithi119.jjbxxd(active, since 64s), standbys: smithi186.vtjgwg 2024-04-22T15:16:23.798 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:23 smithi186 ceph-mon[24041]: mgrmap e37: smithi119.jjbxxd(active, since 64s), standbys: smithi186.vtjgwg 2024-04-22T15:16:24.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 146527 12.55 MB/sec execute 153 sec latency 52.082 ms 2024-04-22T15:16:24.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: Active manager daemon smithi119.jjbxxd restarted 2024-04-22T15:16:24.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: Activating manager daemon smithi119.jjbxxd 2024-04-22T15:16:24.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T15:16:24.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: mgrmap e38: smithi119.jjbxxd(active, starting, since 0.00713799s), standbys: smithi186.vtjgwg 2024-04-22T15:16:24.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:16:24.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:16:24.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:16:24.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:16:24.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi119.jjbxxd", "id": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:16:24.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi186.vtjgwg", "id": "smithi186.vtjgwg"}]: dispatch 2024-04-22T15:16:24.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:16:24.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:16:24.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:16:24.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:16:24.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:16:24.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:16:24.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:16:24.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:16:24.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:16:24.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:16:24.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:16:24.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:24 smithi119 ceph-mon[63156]: Manager daemon smithi119.jjbxxd is now available 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: Active manager daemon smithi119.jjbxxd restarted 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: Activating manager daemon smithi119.jjbxxd 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: mgrmap e38: smithi119.jjbxxd(active, starting, since 0.00713799s), standbys: smithi186.vtjgwg 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi119.jjbxxd", "id": "smithi119.jjbxxd"}]: dispatch 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr metadata", "who": "smithi186.vtjgwg", "id": "smithi186.vtjgwg"}]: dispatch 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:16:24.948 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:16:24.949 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:16:24.949 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:16:24.949 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:16:24.949 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:16:24.949 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:16:24.949 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:16:24.949 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:16:24.949 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:24 smithi186 ceph-mon[24041]: Manager daemon smithi119.jjbxxd is now available 2024-04-22T15:16:25.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 146866 12.53 MB/sec execute 154 sec latency 54.955 ms 2024-04-22T15:16:25.704 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:16:25.705 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:16:25.705 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:16:25.705 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:16:25.705 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:25 smithi119 ceph-mon[63156]: mgrmap e39: smithi119.jjbxxd(active, since 1.01274s), standbys: smithi186.vtjgwg 2024-04-22T15:16:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:25 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:16:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:25 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:16:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:25 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:16:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:25 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi119.jjbxxd/trash_purge_schedule"}]: dispatch 2024-04-22T15:16:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:25 smithi186 ceph-mon[24041]: mgrmap e39: smithi119.jjbxxd(active, since 1.01274s), standbys: smithi186.vtjgwg 2024-04-22T15:16:26.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 147288 12.51 MB/sec execute 155 sec latency 42.951 ms 2024-04-22T15:16:26.718 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:26 smithi119 ceph-mon[63156]: [22/Apr/2024:15:16:25] ENGINE Bus STARTING 2024-04-22T15:16:26.718 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:26 smithi119 ceph-mon[63156]: [22/Apr/2024:15:16:25] ENGINE Serving on https://172.21.15.119:7150 2024-04-22T15:16:26.718 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:26 smithi119 ceph-mon[63156]: [22/Apr/2024:15:16:25] ENGINE Serving on http://172.21.15.119:8765 2024-04-22T15:16:26.718 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:26 smithi119 ceph-mon[63156]: [22/Apr/2024:15:16:25] ENGINE Bus STARTED 2024-04-22T15:16:26.718 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:26 smithi119 ceph-mon[63156]: pgmap v3: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-22T15:16:27.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:26 smithi186 ceph-mon[24041]: [22/Apr/2024:15:16:25] ENGINE Bus STARTING 2024-04-22T15:16:27.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:26 smithi186 ceph-mon[24041]: [22/Apr/2024:15:16:25] ENGINE Serving on https://172.21.15.119:7150 2024-04-22T15:16:27.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:26 smithi186 ceph-mon[24041]: [22/Apr/2024:15:16:25] ENGINE Serving on http://172.21.15.119:8765 2024-04-22T15:16:27.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:26 smithi186 ceph-mon[24041]: [22/Apr/2024:15:16:25] ENGINE Bus STARTED 2024-04-22T15:16:27.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:26 smithi186 ceph-mon[24041]: pgmap v3: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-22T15:16:27.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 147802 12.52 MB/sec execute 156 sec latency 33.946 ms 2024-04-22T15:16:27.722 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:27 smithi119 ceph-mon[63156]: pgmap v4: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-22T15:16:27.722 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:27 smithi119 ceph-mon[63156]: mgrmap e40: smithi119.jjbxxd(active, since 2s), standbys: smithi186.vtjgwg 2024-04-22T15:16:28.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:27 smithi186 ceph-mon[24041]: pgmap v4: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-22T15:16:28.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:27 smithi186 ceph-mon[24041]: mgrmap e40: smithi119.jjbxxd(active, since 2s), standbys: smithi186.vtjgwg 2024-04-22T15:16:28.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 148257 12.50 MB/sec execute 157 sec latency 71.138 ms 2024-04-22T15:16:29.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 148622 12.50 MB/sec execute 158 sec latency 53.503 ms 2024-04-22T15:16:29.782 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:29 smithi119 ceph-mon[63156]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-22T15:16:29.783 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:29 smithi119 ceph-mon[63156]: mgrmap e41: smithi119.jjbxxd(active, since 4s), standbys: smithi186.vtjgwg 2024-04-22T15:16:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:29 smithi186 ceph-mon[24041]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-22T15:16:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:29 smithi186 ceph-mon[24041]: mgrmap e41: smithi119.jjbxxd(active, since 4s), standbys: smithi186.vtjgwg 2024-04-22T15:16:30.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 148910 12.48 MB/sec execute 159 sec latency 52.459 ms 2024-04-22T15:16:31.330 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:31.331 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:31.331 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:31 smithi119 ceph-mon[63156]: pgmap v6: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 50 KiB/s rd, 2.9 MiB/s wr, 272 op/s 2024-04-22T15:16:31.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 149241 12.49 MB/sec execute 160 sec latency 55.635 ms 2024-04-22T15:16:31.524 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:31 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:31.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:31 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:31.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:31 smithi186 ceph-mon[24041]: pgmap v6: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 50 KiB/s rd, 2.9 MiB/s wr, 272 op/s 2024-04-22T15:16:32.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 149543 12.47 MB/sec execute 161 sec latency 82.903 ms 2024-04-22T15:16:32.614 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:32.615 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:32.740 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:32 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:32.740 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:32 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:33.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 149983 12.43 MB/sec execute 162 sec latency 33.146 ms 2024-04-22T15:16:33.626 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:33 smithi119 ceph-mon[63156]: pgmap v7: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 4.4 MiB/s wr, 445 op/s 2024-04-22T15:16:33.667 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:33 smithi186 ceph-mon[24041]: pgmap v7: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 4.4 MiB/s wr, 445 op/s 2024-04-22T15:16:34.468 INFO:teuthology.orchestra.run.smithi119.stdout: 5 150427 12.44 MB/sec execute 163 sec latency 26.757 ms 2024-04-22T15:16:35.406 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:35.407 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:35.407 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:35 smithi119 ceph-mon[63156]: pgmap v8: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 3.4 MiB/s wr, 346 op/s 2024-04-22T15:16:35.467 INFO:teuthology.orchestra.run.smithi119.stdout: 5 150848 12.41 MB/sec execute 164 sec latency 52.869 ms 2024-04-22T15:16:35.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:35 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:35.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:35 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:35.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:35 smithi186 ceph-mon[24041]: pgmap v8: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 3.4 MiB/s wr, 346 op/s 2024-04-22T15:16:36.468 INFO:teuthology.orchestra.run.smithi119.stdout: 5 151271 12.40 MB/sec execute 165 sec latency 33.553 ms 2024-04-22T15:16:36.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:36.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:36.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:36.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:36.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:36.689 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:16:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:36 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:36 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:36 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:36 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:36 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:36 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:16:37.468 INFO:teuthology.orchestra.run.smithi119.stdout: 5 151686 12.39 MB/sec execute 166 sec latency 40.641 ms 2024-04-22T15:16:37.698 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: Updating smithi119:/etc/ceph/ceph.conf 2024-04-22T15:16:37.699 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: Updating smithi186:/etc/ceph/ceph.conf 2024-04-22T15:16:37.699 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: Updating smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:16:37.699 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: Updating smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:16:37.699 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: pgmap v9: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.3 MiB/s wr, 546 op/s 2024-04-22T15:16:37.699 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:16:37.699 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: Updating smithi119:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:16:37.699 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:37.699 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:37.699 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:37.699 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:37.699 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:37.699 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:37 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: Updating smithi119:/etc/ceph/ceph.conf 2024-04-22T15:16:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: Updating smithi186:/etc/ceph/ceph.conf 2024-04-22T15:16:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: Updating smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:16:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: Updating smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:16:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: pgmap v9: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.3 MiB/s wr, 546 op/s 2024-04-22T15:16:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:16:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: Updating smithi119:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:16:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:37.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:37.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:37.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:37.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:37 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:38.468 INFO:teuthology.orchestra.run.smithi119.stdout: 5 152065 12.37 MB/sec execute 167 sec latency 47.016 ms 2024-04-22T15:16:38.718 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:38 smithi119 ceph-mon[63156]: Updating smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:16:38.718 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:38 smithi119 ceph-mon[63156]: Updating smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:16:38.718 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:16:38.719 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:16:38.719 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:16:38.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:38 smithi186 ceph-mon[24041]: Updating smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:16:38.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:38 smithi186 ceph-mon[24041]: Updating smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:16:38.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:38 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:16:38.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:38 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:16:38.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:38 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:16:39.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 152411 12.37 MB/sec execute 168 sec latency 48.822 ms 2024-04-22T15:16:39.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:39 smithi119 ceph-mon[63156]: pgmap v10: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 4.9 MiB/s wr, 500 op/s 2024-04-22T15:16:39.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:39.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:16:39.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:16:39.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:39.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:39 smithi186 ceph-mon[24041]: pgmap v10: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 4.9 MiB/s wr, 500 op/s 2024-04-22T15:16:39.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:39 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:39.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:39 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:16:39.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:39 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:16:39.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:39 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:40.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 152842 12.41 MB/sec execute 169 sec latency 56.476 ms 2024-04-22T15:16:40.758 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:40 smithi119 ceph-mon[63156]: Upgrade: Updating mon.smithi186 2024-04-22T15:16:40.758 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:40 smithi119 ceph-mon[63156]: Deploying daemon mon.smithi186 on smithi186 2024-04-22T15:16:40.758 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:40 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:16:40.774 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:40 smithi186 ceph-mon[24041]: Upgrade: Updating mon.smithi186 2024-04-22T15:16:40.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:40 smithi186 ceph-mon[24041]: Deploying daemon mon.smithi186 on smithi186 2024-04-22T15:16:40.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:40 smithi186 ceph-mon[24041]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:16:41.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 153154 12.38 MB/sec execute 170 sec latency 62.472 ms 2024-04-22T15:16:41.780 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:41 smithi119 ceph-mon[63156]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 6.5 MiB/s wr, 695 op/s 2024-04-22T15:16:42.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:41 smithi186 systemd[1]: Stopping Ceph mon.smithi186 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:16:42.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:41 smithi186 ceph-mon[24041]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 6.5 MiB/s wr, 695 op/s 2024-04-22T15:16:42.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 153461 12.33 MB/sec execute 171 sec latency 41.480 ms 2024-04-22T15:16:42.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:42 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186[24028]: 2024-04-22T15:16:42.164+0000 7f5bc53e6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi186 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:16:42.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:42 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186[24028]: 2024-04-22T15:16:42.164+0000 7f5bc53e6700 -1 mon.smithi186@1(peon) e2 *** Got Signal Terminated *** 2024-04-22T15:16:42.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:42 smithi186 podman[49642]: 2024-04-22 15:16:42.215623384 +0000 UTC m=+0.508081812 container died a804e52c2c5e51a31e5bae05bff2dc3f62ac15bc04ca460027b19e77186eccc8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186, GIT_CLEAN=True, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, vcs-type=git, RELEASE=HEAD, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-22T15:16:42.995 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:42 smithi186 podman[49642]: 2024-04-22 15:16:42.740727534 +0000 UTC m=+1.033185962 container cleanup a804e52c2c5e51a31e5bae05bff2dc3f62ac15bc04ca460027b19e77186eccc8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , release=754, vcs-type=git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, name=centos-stream, ceph=True, distribution-scope=public) 2024-04-22T15:16:42.996 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:42 smithi186 bash[49642]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186 2024-04-22T15:16:43.258 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:42 smithi186 podman[49655]: 2024-04-22 15:16:42.992837819 +0000 UTC m=+0.775852813 container remove a804e52c2c5e51a31e5bae05bff2dc3f62ac15bc04ca460027b19e77186eccc8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vendor=Red Hat, Inc., GIT_CLEAN=True, name=centos-stream, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, RELEASE=HEAD) 2024-04-22T15:16:43.312 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:16:43.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 153700 12.28 MB/sec execute 172 sec latency 52.850 ms 2024-04-22T15:16:43.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:43 smithi186 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi186.service: Deactivated successfully. 2024-04-22T15:16:43.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:43 smithi186 systemd[1]: Stopped Ceph mon.smithi186 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:16:43.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:43 smithi186 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi186.service: Consumed 7.059s CPU time. 2024-04-22T15:16:43.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:43 smithi186 systemd[1]: Starting Ceph mon.smithi186 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:16:44.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:43 smithi186 podman[49786]: 2024-04-22 15:16:43.635077251 +0000 UTC m=+0.021420405 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:16:44.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:43 smithi186 podman[49786]: 2024-04-22 15:16:43.77809356 +0000 UTC m=+0.164436695 container create 38867a5eb7be29953953fe64f1bac4c443ee60df22483bc840632ce6c0d552f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2) 2024-04-22T15:16:44.311 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:16:44.311 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (2m) 9s ago 11m 15.8M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:16:44.311 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (11m) 9s ago 11m 6405k - 17.2.0 e1d6a67b021e 37eb54a66662 2024-04-22T15:16:44.311 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (10m) 14s ago 10m 6405k - 17.2.0 e1d6a67b021e ee37991f5d20 2024-04-22T15:16:44.311 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (2m) 9s ago 11m 45.4M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:16:44.311 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (5m) 9s ago 5m 38.3M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:16:44.311 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (5m) 14s ago 5m 13.8M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:16:44.311 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (106s) 9s ago 12m 479M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (51s) 14s ago 10m 448M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (31s) 9s ago 12m 33.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 starting - - - 2048M 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (5m) 9s ago 5m 270M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (5m) 14s ago 5m 83.8M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (2m) 9s ago 11m 12.4M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (115s) 14s ago 10m 16.7M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (9m) 9s ago 9m 508M 1936M 17.2.0 e1d6a67b021e 9999e9684eba 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (9m) 9s ago 9m 444M 1936M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (8m) 9s ago 8m 327M 1936M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (8m) 9s ago 8m 472M 1936M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (8m) 14s ago 8m 480M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (7m) 14s ago 7m 565M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:16:44.312 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (7m) 14s ago 7m 472M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:16:44.313 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (6m) 14s ago 6m 376M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:16:44.313 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (58s) 9s ago 10m 60.3M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:16:44.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 154284 12.31 MB/sec execute 173 sec latency 35.673 ms 2024-04-22T15:16:44.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 podman[49786]: 2024-04-22 15:16:44.023315923 +0000 UTC m=+0.409659067 container init 38867a5eb7be29953953fe64f1bac4c443ee60df22483bc840632ce6c0d552f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:16:44.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 podman[49786]: 2024-04-22 15:16:44.027706905 +0000 UTC m=+0.414050043 container start 38867a5eb7be29953953fe64f1bac4c443ee60df22483bc840632ce6c0d552f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev), process ceph-mon, pid 2 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: pidfile_write: ignore empty --pid-file 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: load: jerasure load: lrc 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: RocksDB version: 7.9.2 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Git sha 0 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Compile date 2024-04-22 12:55:29 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: DB SUMMARY 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: DB Session ID: 0IJ5KWHRFTI52ED8IVU5 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: CURRENT file: CURRENT 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: MANIFEST file: MANIFEST-000009 size: 904 Bytes 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi186/store.db dir, Total Num: 1, files: 000024.sst 2024-04-22T15:16:44.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi186/store.db: 000022.log size: 5663304 ; 2024-04-22T15:16:44.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.error_if_exists: 0 2024-04-22T15:16:44.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.create_if_missing: 0 2024-04-22T15:16:44.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.paranoid_checks: 1 2024-04-22T15:16:44.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-22T15:16:44.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T15:16:44.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-22T15:16:44.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.env: 0x5652c22c8cc0 2024-04-22T15:16:44.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.fs: PosixFileSystem 2024-04-22T15:16:44.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.info_log: 0x5652c4849aa0 2024-04-22T15:16:44.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T15:16:44.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.statistics: (nil) 2024-04-22T15:16:44.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.use_fsync: 0 2024-04-22T15:16:44.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_log_file_size: 0 2024-04-22T15:16:44.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T15:16:44.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T15:16:44.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T15:16:44.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T15:16:44.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.allow_fallocate: 1 2024-04-22T15:16:44.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.use_direct_reads: 0 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.db_log_dir: 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.wal_dir: 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.write_buffer_manager: 0x5652c47dd0e0 2024-04-22T15:16:44.529 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.unordered_write: 0 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.row_cache: None 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.wal_filter: None 2024-04-22T15:16:44.530 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.two_write_queues: 0 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.wal_compression: 0 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.atomic_flush: 0 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.log_readahead_size: 0 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T15:16:44.531 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_background_jobs: 2 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_background_compactions: -1 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_subcompactions: 1 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_open_files: -1 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T15:16:44.532 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_background_flushes: -1 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Compression algorithms supported: 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: kZSTD supported: 0 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: kXpressCompression supported: 0 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: kZlibCompression supported: 1 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: kSnappyCompression supported: 1 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: kLZ4Compression supported: 1 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: kBZip2Compression supported: 0 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi186/store.db/MANIFEST-000009 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-22T15:16:44.533 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.merge_operator: 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_filter: None 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5652c4849c80) 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: cache_index_and_filter_blocks: 1 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: pin_top_level_index_and_filter: 1 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: index_type: 0 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: data_block_index_type: 0 2024-04-22T15:16:44.534 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: index_shortening: 1 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: checksum: 4 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: no_block_cache: 0 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: block_cache: 0x5652c480f1f0 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: block_cache_name: BinnedLRUCache 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: block_cache_options: 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: capacity : 536870912 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: num_shard_bits : 4 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: strict_capacity_limit : 0 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: high_pri_pool_ratio: 0.000 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: block_cache_compressed: (nil) 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: persistent_cache: (nil) 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: block_size: 4096 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: block_size_deviation: 10 2024-04-22T15:16:44.535 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: block_restart_interval: 16 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: index_block_restart_interval: 1 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: metadata_block_size: 4096 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: partition_filters: 0 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: use_delta_encoding: 1 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: filter_policy: bloomfilter 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: whole_key_filtering: 1 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: verify_compression: 0 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: read_amp_bytes_per_bit: 0 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: format_version: 5 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: enable_index_compression: 1 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: block_align: 0 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: max_auto_readahead_size: 262144 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: prepopulate_block_cache: 0 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: initial_auto_readahead_size: 8192 2024-04-22T15:16:44.536 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: num_file_reads_for_auto_readahead: 2 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compression: NoCompression 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.num_levels: 7 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T15:16:44.537 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T15:16:44.538 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T15:16:44.539 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.arena_block_size: 1048576 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T15:16:44.540 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.inplace_update_support: 0 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.bloom_locality: 0 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.max_successive_merges: 0 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.ttl: 2592000 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T15:16:44.541 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-22T15:16:44.542 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-22T15:16:44.542 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.enable_blob_files: false 2024-04-22T15:16:44.542 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.min_blob_size: 0 2024-04-22T15:16:44.542 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.blob_file_starting_level: 0 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi186/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12246, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d1171d63-6607-4d79-b64c-8f9b0bfce6df 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713799004063561, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-04-22T15:16:44.543 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713799004078048, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 3482040, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12251, "largest_seqno": 13724, "table_properties": {"data_size": 3475394, "index_size": 4129, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 13728, "raw_average_key_size": 23, "raw_value_size": 3462505, "raw_average_value_size": 5918, "num_data_blocks": 188, "num_entries": 585, "num_filter_entries": 585, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1713799004, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d1171d63-6607-4d79-b64c-8f9b0bfce6df", "db_session_id": "0IJ5KWHRFTI52ED8IVU5", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713799004078189, "job": 1, "event": "recovery_finished"} 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi186/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5652c4924000 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: DB pointer 0x5652c490e000 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: ** DB Stats ** 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:16:44.544 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: ** Compaction Stats [default] ** 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: L0 1/0 3.32 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 600.6 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: L6 1/0 7.38 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Sum 2/0 10.70 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 600.6 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 600.6 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: ** Compaction Stats [default] ** 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 600.6 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: 2024-04-22T15:16:44.545 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Cumulative compaction: 0.00 GB write, 167.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Interval compaction: 0.00 GB write, 167.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Block cache BinnedLRUCache@0x5652c480f1f0#2 capacity: 512.00 MB usage: 15.14 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,10.56 KB,0.00201464%) IndexBlock(2,20.66 KB,0.00393987%) Misc(1,0.00 KB,0%) 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: starting mon.smithi186 rank 1 at public addrs [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] at bind addrs [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi186 fsid 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: mon.smithi186@-1(???) e2 preinit fsid 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:16:44.546 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: mon.smithi186@-1(???).mds e7 new map 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: mon.smithi186@-1(???).mds e7 print_map 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: e7 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: legacy client fscid: 1 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Filesystem 'foofs' (1) 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: fs_name foofs 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: epoch 5 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: created 2024-04-22T15:11:14.746120+0000 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: modified 2024-04-22T15:11:20.769028+0000 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: tableserver 0 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: root 0 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: session_timeout 60 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: session_autoclose 300 2024-04-22T15:16:44.547 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: max_file_size 1099511627776 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: max_xattr_size 65536 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: required_client_features {} 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: last_failure 0 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: last_failure_osd_epoch 0 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: max_mds 1 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: in 0 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: up {0=14490} 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: failed 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: damaged 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: stopped 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: data_pools [3] 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: metadata_pool 2 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: inline_data disabled 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: balancer 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: bal_rank_mask -1 2024-04-22T15:16:44.548 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: standby_count_wanted 1 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: qdb_cluster leader: 0 members: 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: [mds.foofs.smithi119.ncbqaf{0:14490} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.119:6834/3732854443,v1:172.21.15.119:6835/3732854443] compat {c=[1],r=[1],i=[7ff]}] 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: Standby daemons: 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout: [mds.foofs.smithi186.dglszf{-1:24281} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.186:6832/2825961187,v1:172.21.15.186:6833/2825961187] compat {c=[1],r=[1],i=[7ff]}] 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: mon.smithi186@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: mon.smithi186@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: mon.smithi186@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: mon.smithi186@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 ceph-mon[49804]: mon.smithi186@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 bash[49786]: 38867a5eb7be29953953fe64f1bac4c443ee60df22483bc840632ce6c0d552f7 2024-04-22T15:16:44.549 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:44 smithi186 systemd[1]: Started Ceph mon.smithi186 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:16:44.942 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:16:44.942 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:16:44.942 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T15:16:44.942 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:16:44.942 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:16:44.942 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:16:44.942 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:16:44.942 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:16:44.942 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:16:44.942 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:16:44.943 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:16:44.943 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:16:44.943 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:16:44.943 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:16:44.943 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:16:44.943 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-04-22T15:16:44.943 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:16:44.943 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:16:44.943 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:16:45.452 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:16:45.452 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:16:45.452 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:16:45.452 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:16:45.452 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:16:45.452 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr" 2024-04-22T15:16:45.452 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:16:45.452 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "3/23 daemons upgraded", 2024-04-22T15:16:45.452 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading mon daemons", 2024-04-22T15:16:45.452 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:16:45.453 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:16:45.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 154825 12.32 MB/sec execute 174 sec latency 58.094 ms 2024-04-22T15:16:46.164 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:16:46.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 155334 12.30 MB/sec execute 175 sec latency 51.724 ms 2024-04-22T15:16:46.858 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:16:46.858 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: mgrmap e41: smithi119.jjbxxd(active, since 21s), standbys: smithi186.vtjgwg 2024-04-22T15:16:46.858 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: overall HEALTH_OK 2024-04-22T15:16:46.858 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: mon.smithi119 calling monitor election 2024-04-22T15:16:46.858 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: mon.smithi186 calling monitor election 2024-04-22T15:16:46.858 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:16:46.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: mon.smithi119 is new leader, mons smithi119,smithi186 in quorum (ranks 0,1) 2024-04-22T15:16:46.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:16:46.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: monmap epoch 3 2024-04-22T15:16:46.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: fsid 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:16:46.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: last_changed 2024-04-22T15:16:45.676473+0000 2024-04-22T15:16:46.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: created 2024-04-22T15:03:50.504762+0000 2024-04-22T15:16:46.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: min_mon_release 19 (squid) 2024-04-22T15:16:46.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: election_strategy: 1 2024-04-22T15:16:46.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: 0: [v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0] mon.smithi119 2024-04-22T15:16:46.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: 1: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.smithi186 2024-04-22T15:16:46.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 1 up:standby 2024-04-22T15:16:46.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T15:16:46.860 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: mgrmap e41: smithi119.jjbxxd(active, since 21s), standbys: smithi186.vtjgwg 2024-04-22T15:16:46.860 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: overall HEALTH_OK 2024-04-22T15:16:46.860 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:46.860 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:46.860 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:16:46.860 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:46 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/1696997767' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:16:47.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:16:47.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: mgrmap e41: smithi119.jjbxxd(active, since 21s), standbys: smithi186.vtjgwg 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: overall HEALTH_OK 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: mon.smithi119 calling monitor election 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: mon.smithi186 calling monitor election 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi119"}]: dispatch 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: mon.smithi119 is new leader, mons smithi119,smithi186 in quorum (ranks 0,1) 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mon metadata", "id": "smithi186"}]: dispatch 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: monmap epoch 3 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: fsid 74739db6-00b9-11ef-bc93-c7b262605968 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: last_changed 2024-04-22T15:16:45.676473+0000 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: created 2024-04-22T15:03:50.504762+0000 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: min_mon_release 19 (squid) 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: election_strategy: 1 2024-04-22T15:16:47.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: 0: [v2:172.21.15.119:3300/0,v1:172.21.15.119:6789/0] mon.smithi119 2024-04-22T15:16:47.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: 1: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.smithi186 2024-04-22T15:16:47.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 1 up:standby 2024-04-22T15:16:47.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T15:16:47.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: mgrmap e41: smithi119.jjbxxd(active, since 21s), standbys: smithi186.vtjgwg 2024-04-22T15:16:47.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: overall HEALTH_OK 2024-04-22T15:16:47.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:47.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:47.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:16:47.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:46 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/1696997767' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:16:47.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 155892 12.35 MB/sec execute 176 sec latency 43.857 ms 2024-04-22T15:16:47.877 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:47 smithi119 ceph-mon[63156]: pgmap v14: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 726 op/s 2024-04-22T15:16:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:47 smithi186 ceph-mon[49804]: pgmap v14: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 726 op/s 2024-04-22T15:16:48.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 156393 12.39 MB/sec execute 177 sec latency 46.469 ms 2024-04-22T15:16:49.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 156809 12.37 MB/sec execute 178 sec latency 43.372 ms 2024-04-22T15:16:49.823 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:49 smithi119 ceph-mon[63156]: pgmap v15: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 485 op/s 2024-04-22T15:16:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:49 smithi186 ceph-mon[49804]: pgmap v15: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 485 op/s 2024-04-22T15:16:50.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 157090 12.32 MB/sec execute 179 sec latency 37.284 ms 2024-04-22T15:16:51.092 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:50 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:51.092 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:50 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:51.092 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:50 smithi119 ceph-mon[63156]: pgmap v16: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.2 MiB/s wr, 550 op/s 2024-04-22T15:16:51.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:50 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:51.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:50 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:51.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:50 smithi186 ceph-mon[49804]: pgmap v16: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.2 MiB/s wr, 550 op/s 2024-04-22T15:16:51.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 157520 12.32 MB/sec execute 180 sec latency 27.772 ms 2024-04-22T15:16:52.152 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:52 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:52.152 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:52 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:52.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 157869 12.29 MB/sec execute 181 sec latency 79.510 ms 2024-04-22T15:16:52.524 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:52 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:52.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:52 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:53.165 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:53 smithi119 ceph-mon[63156]: pgmap v17: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.0 MiB/s wr, 565 op/s 2024-04-22T15:16:53.213 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:53 smithi186 ceph-mon[49804]: pgmap v17: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.0 MiB/s wr, 565 op/s 2024-04-22T15:16:53.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 158273 12.29 MB/sec execute 182 sec latency 43.367 ms 2024-04-22T15:16:54.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 158632 12.25 MB/sec execute 183 sec latency 37.284 ms 2024-04-22T15:16:55.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 158955 12.23 MB/sec execute 184 sec latency 33.692 ms 2024-04-22T15:16:55.951 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:55 smithi119 ceph-mon[63156]: pgmap v18: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 525 op/s 2024-04-22T15:16:55.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:16:55.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:55.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:55.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:55.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:16:55.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:55.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:55.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:16:55.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:16:55.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:55 smithi186 ceph-mon[49804]: pgmap v18: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 525 op/s 2024-04-22T15:16:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:16:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:16:56.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:56.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:56.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:16:56.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:16:56.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:56.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 159306 12.23 MB/sec execute 185 sec latency 41.283 ms 2024-04-22T15:16:56.976 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:56 smithi119 ceph-mon[63156]: Reconfiguring mon.smithi119 (monmap changed)... 2024-04-22T15:16:56.976 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:56 smithi119 ceph-mon[63156]: Reconfiguring daemon mon.smithi119 on smithi119 2024-04-22T15:16:57.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:56 smithi186 ceph-mon[49804]: Reconfiguring mon.smithi119 (monmap changed)... 2024-04-22T15:16:57.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:56 smithi186 ceph-mon[49804]: Reconfiguring daemon mon.smithi119 on smithi119 2024-04-22T15:16:57.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 159609 12.22 MB/sec execute 186 sec latency 44.421 ms 2024-04-22T15:16:57.739 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:57 smithi119 ceph-mon[63156]: pgmap v19: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 755 op/s 2024-04-22T15:16:58.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:57 smithi186 ceph-mon[49804]: pgmap v19: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 755 op/s 2024-04-22T15:16:58.475 INFO:teuthology.orchestra.run.smithi119.stdout: 5 160074 12.25 MB/sec execute 187 sec latency 66.397 ms 2024-04-22T15:16:59.476 INFO:teuthology.orchestra.run.smithi119.stdout: 5 160497 12.23 MB/sec execute 188 sec latency 35.492 ms 2024-04-22T15:16:59.509 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:59.509 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:59.509 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:59 smithi119 ceph-mon[63156]: Reconfiguring mgr.smithi119.jjbxxd (monmap changed)... 2024-04-22T15:16:59.509 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi119.jjbxxd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:16:59.510 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:16:59.510 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:59.510 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:59 smithi119 ceph-mon[63156]: Reconfiguring daemon mgr.smithi119.jjbxxd on smithi119 2024-04-22T15:16:59.510 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:16:59 smithi119 ceph-mon[63156]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.1 MiB/s wr, 504 op/s 2024-04-22T15:16:59.524 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:59.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:16:59.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:59 smithi186 ceph-mon[49804]: Reconfiguring mgr.smithi119.jjbxxd (monmap changed)... 2024-04-22T15:16:59.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi119.jjbxxd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:16:59.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:16:59.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:59.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:59 smithi186 ceph-mon[49804]: Reconfiguring daemon mgr.smithi119.jjbxxd on smithi119 2024-04-22T15:16:59.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:16:59 smithi186 ceph-mon[49804]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.1 MiB/s wr, 504 op/s 2024-04-22T15:17:00.476 INFO:teuthology.orchestra.run.smithi119.stdout: 5 160908 12.21 MB/sec execute 189 sec latency 33.351 ms 2024-04-22T15:17:01.476 INFO:teuthology.orchestra.run.smithi119.stdout: 5 161396 12.21 MB/sec execute 190 sec latency 36.172 ms 2024-04-22T15:17:01.800 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:01 smithi119 ceph-mon[63156]: pgmap v21: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 643 op/s 2024-04-22T15:17:01.800 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:01 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:01.800 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:01 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:01.800 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:01 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi119", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:17:01.801 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:01 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:01 smithi186 ceph-mon[49804]: pgmap v21: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 643 op/s 2024-04-22T15:17:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:01 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:01 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:01 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi119", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:17:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:01 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:02.478 INFO:teuthology.orchestra.run.smithi119.stdout: 5 161978 12.23 MB/sec execute 191 sec latency 28.296 ms 2024-04-22T15:17:02.807 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:02 smithi119 ceph-mon[63156]: Reconfiguring crash.smithi119 (monmap changed)... 2024-04-22T15:17:02.808 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:02 smithi119 ceph-mon[63156]: Reconfiguring daemon crash.smithi119 on smithi119 2024-04-22T15:17:03.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:02 smithi186 ceph-mon[49804]: Reconfiguring crash.smithi119 (monmap changed)... 2024-04-22T15:17:03.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:02 smithi186 ceph-mon[49804]: Reconfiguring daemon crash.smithi119 on smithi119 2024-04-22T15:17:03.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 162547 12.23 MB/sec execute 192 sec latency 32.916 ms 2024-04-22T15:17:03.816 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:03 smithi119 ceph-mon[63156]: pgmap v22: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 682 op/s 2024-04-22T15:17:04.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:03 smithi186 ceph-mon[49804]: pgmap v22: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 682 op/s 2024-04-22T15:17:04.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 163038 12.26 MB/sec execute 193 sec latency 40.045 ms 2024-04-22T15:17:05.335 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:05 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:05.335 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:05 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:05.335 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:05 smithi119 ceph-mon[63156]: Reconfiguring osd.0 (monmap changed)... 2024-04-22T15:17:05.335 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:05 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:17:05.335 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:05 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:05.335 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:05 smithi119 ceph-mon[63156]: Reconfiguring daemon osd.0 on smithi119 2024-04-22T15:17:05.335 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:05 smithi119 ceph-mon[63156]: pgmap v23: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-04-22T15:17:05.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 163406 12.28 MB/sec execute 194 sec latency 34.330 ms 2024-04-22T15:17:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:05 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:05 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:05 smithi186 ceph-mon[49804]: Reconfiguring osd.0 (monmap changed)... 2024-04-22T15:17:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:05 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:17:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:05 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:05 smithi186 ceph-mon[49804]: Reconfiguring daemon osd.0 on smithi119 2024-04-22T15:17:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:05 smithi186 ceph-mon[49804]: pgmap v23: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-04-22T15:17:06.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 163771 12.25 MB/sec execute 195 sec latency 45.671 ms 2024-04-22T15:17:07.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 164062 12.22 MB/sec execute 196 sec latency 43.570 ms 2024-04-22T15:17:07.884 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:07 smithi119 ceph-mon[63156]: pgmap v24: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 768 op/s 2024-04-22T15:17:07.884 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:07.885 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:07.885 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:17:07.885 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:07 smithi186 ceph-mon[49804]: pgmap v24: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 768 op/s 2024-04-22T15:17:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:17:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:08.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 164299 12.18 MB/sec execute 197 sec latency 39.955 ms 2024-04-22T15:17:08.914 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:08 smithi119 ceph-mon[63156]: Reconfiguring osd.1 (monmap changed)... 2024-04-22T15:17:08.914 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:08 smithi119 ceph-mon[63156]: Reconfiguring daemon osd.1 on smithi119 2024-04-22T15:17:09.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:08 smithi186 ceph-mon[49804]: Reconfiguring osd.1 (monmap changed)... 2024-04-22T15:17:09.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:08 smithi186 ceph-mon[49804]: Reconfiguring daemon osd.1 on smithi119 2024-04-22T15:17:09.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 164606 12.17 MB/sec execute 198 sec latency 43.560 ms 2024-04-22T15:17:09.939 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:09 smithi119 ceph-mon[63156]: pgmap v25: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 538 op/s 2024-04-22T15:17:10.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:09 smithi186 ceph-mon[49804]: pgmap v25: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 538 op/s 2024-04-22T15:17:10.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 164916 12.14 MB/sec execute 199 sec latency 64.505 ms 2024-04-22T15:17:10.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:17:10.953 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:10.953 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:10.953 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:17:10.953 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:17:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:17:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:11.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 165565 12.16 MB/sec execute 200 sec latency 32.982 ms 2024-04-22T15:17:11.958 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:11 smithi119 ceph-mon[63156]: Reconfiguring osd.2 (monmap changed)... 2024-04-22T15:17:11.958 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:11 smithi119 ceph-mon[63156]: Reconfiguring daemon osd.2 on smithi119 2024-04-22T15:17:11.959 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:11 smithi119 ceph-mon[63156]: pgmap v26: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.9 MiB/s wr, 660 op/s 2024-04-22T15:17:12.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:11 smithi186 ceph-mon[49804]: Reconfiguring osd.2 (monmap changed)... 2024-04-22T15:17:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:11 smithi186 ceph-mon[49804]: Reconfiguring daemon osd.2 on smithi119 2024-04-22T15:17:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:11 smithi186 ceph-mon[49804]: pgmap v26: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.9 MiB/s wr, 660 op/s 2024-04-22T15:17:12.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 166220 12.18 MB/sec execute 201 sec latency 33.283 ms 2024-04-22T15:17:13.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 166982 12.27 MB/sec execute 202 sec latency 27.226 ms 2024-04-22T15:17:13.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:13 smithi119 ceph-mon[63156]: pgmap v27: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.7 MiB/s wr, 598 op/s 2024-04-22T15:17:13.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:13 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:13.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:13 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:13.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:13 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:17:13.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:13 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:14.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:13 smithi186 ceph-mon[49804]: pgmap v27: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.7 MiB/s wr, 598 op/s 2024-04-22T15:17:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:13 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:13 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:13 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:17:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:13 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:14.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 167656 12.28 MB/sec execute 203 sec latency 28.472 ms 2024-04-22T15:17:14.948 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:14 smithi119 ceph-mon[63156]: Reconfiguring osd.3 (monmap changed)... 2024-04-22T15:17:14.949 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:14 smithi119 ceph-mon[63156]: Reconfiguring daemon osd.3 on smithi119 2024-04-22T15:17:15.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:14 smithi186 ceph-mon[49804]: Reconfiguring osd.3 (monmap changed)... 2024-04-22T15:17:15.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:14 smithi186 ceph-mon[49804]: Reconfiguring daemon osd.3 on smithi119 2024-04-22T15:17:15.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 168280 12.28 MB/sec execute 204 sec latency 31.562 ms 2024-04-22T15:17:15.954 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:15 smithi119 ceph-mon[63156]: pgmap v28: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.8 MiB/s wr, 494 op/s 2024-04-22T15:17:16.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:15 smithi186 ceph-mon[49804]: pgmap v28: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.8 MiB/s wr, 494 op/s 2024-04-22T15:17:16.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 168737 12.28 MB/sec execute 205 sec latency 87.477 ms 2024-04-22T15:17:16.713 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:17:17.232 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:16 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:17.232 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:16 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:17.233 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:16 smithi119 ceph-mon[63156]: Reconfiguring mds.foofs.smithi119.ncbqaf (monmap changed)... 2024-04-22T15:17:17.233 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:16 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi119.ncbqaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T15:17:17.233 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:16 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:17.233 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:16 smithi119 ceph-mon[63156]: Reconfiguring daemon mds.foofs.smithi119.ncbqaf on smithi119 2024-04-22T15:17:17.233 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:16 smithi119 ceph-mon[63156]: pgmap v29: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 894 op/s 2024-04-22T15:17:17.233 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:16 smithi119 ceph-mon[63156]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:17.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:16 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:16 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:16 smithi186 ceph-mon[49804]: Reconfiguring mds.foofs.smithi119.ncbqaf (monmap changed)... 2024-04-22T15:17:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:16 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi119.ncbqaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T15:17:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:16 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:16 smithi186 ceph-mon[49804]: Reconfiguring daemon mds.foofs.smithi119.ncbqaf on smithi119 2024-04-22T15:17:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:16 smithi186 ceph-mon[49804]: pgmap v29: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 894 op/s 2024-04-22T15:17:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:16 smithi186 ceph-mon[49804]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:17.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 169059 12.26 MB/sec execute 206 sec latency 46.856 ms 2024-04-22T15:17:17.715 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:17:17.715 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (2m) 43s ago 11m 15.8M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:17:17.715 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (11m) 43s ago 11m 6405k - 17.2.0 e1d6a67b021e 37eb54a66662 2024-04-22T15:17:17.715 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (11m) 27s ago 11m 6405k - 17.2.0 e1d6a67b021e ee37991f5d20 2024-04-22T15:17:17.715 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (2m) 43s ago 11m 45.4M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:17:17.715 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (5m) 43s ago 5m 38.3M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:17:17.715 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (5m) 27s ago 5m 14.0M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:17:17.715 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (2m) 43s ago 13m 479M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:17:17.715 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (84s) 27s ago 11m 449M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (64s) 43s ago 13m 33.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (34s) 27s ago 11m 25.5M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (5m) 43s ago 5m 270M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (5m) 27s ago 5m 84.3M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (2m) 43s ago 11m 12.4M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (2m) 27s ago 11m 16.7M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (10m) 43s ago 10m 508M 1936M 17.2.0 e1d6a67b021e 9999e9684eba 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (9m) 43s ago 9m 444M 1936M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (9m) 43s ago 9m 327M 1936M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (9m) 43s ago 9m 472M 1936M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (8m) 27s ago 8m 501M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (8m) 27s ago 8m 325M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:17:17.716 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (7m) 27s ago 7m 512M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:17:17.717 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (7m) 27s ago 7m 393M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:17:17.717 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (91s) 43s ago 10m 60.3M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:17:18.251 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:18 smithi119 ceph-mon[63156]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:18.251 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:18 smithi119 ceph-mon[63156]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:18.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:18 smithi186 ceph-mon[49804]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:18.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:18 smithi186 ceph-mon[49804]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:18.288 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:17:18.288 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-22T15:17:18.289 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 4 2024-04-22T15:17:18.290 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:17:18.290 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:17:18.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 169396 12.25 MB/sec execute 207 sec latency 65.016 ms 2024-04-22T15:17:18.787 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:17:18.787 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:17:18.787 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:17:18.788 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:17:18.788 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:17:18.788 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:17:18.788 INFO:teuthology.orchestra.run.smithi119.stdout: "mon" 2024-04-22T15:17:18.788 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:17:18.788 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "4/23 daemons upgraded", 2024-04-22T15:17:18.788 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading mon daemons", 2024-04-22T15:17:18.788 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:17:18.788 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:17:19.256 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:19 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/1050152979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:17:19.257 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:19 smithi119 ceph-mon[63156]: pgmap v30: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.0 MiB/s wr, 598 op/s 2024-04-22T15:17:19.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:19 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/1050152979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:17:19.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:19 smithi186 ceph-mon[49804]: pgmap v30: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.0 MiB/s wr, 598 op/s 2024-04-22T15:17:19.385 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:17:19.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 169752 12.23 MB/sec execute 208 sec latency 43.393 ms 2024-04-22T15:17:20.287 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:20 smithi119 ceph-mon[63156]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:20.287 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:20 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:20.288 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:20 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:20.288 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:20 smithi119 ceph-mon[63156]: Reconfiguring crash.smithi186 (monmap changed)... 2024-04-22T15:17:20.288 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:20 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi186", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:17:20.288 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:20 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:20.288 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:20 smithi119 ceph-mon[63156]: Reconfiguring daemon crash.smithi186 on smithi186 2024-04-22T15:17:20.288 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:20 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/3843659618' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:17:20.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 170120 12.23 MB/sec execute 209 sec latency 45.446 ms 2024-04-22T15:17:20.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:20 smithi186 ceph-mon[49804]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:20.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:20 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:20.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:20 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:20.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:20 smithi186 ceph-mon[49804]: Reconfiguring crash.smithi186 (monmap changed)... 2024-04-22T15:17:20.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:20 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi186", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:17:20.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:20 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:20.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:20 smithi186 ceph-mon[49804]: Reconfiguring daemon crash.smithi186 on smithi186 2024-04-22T15:17:20.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:20 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/3843659618' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:17:21.297 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:21 smithi119 ceph-mon[63156]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.9 MiB/s wr, 651 op/s 2024-04-22T15:17:21.297 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:21.297 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:21.297 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi186.vtjgwg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:17:21.298 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:17:21.298 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:21.484 INFO:teuthology.orchestra.run.smithi119.stdout: 5 170459 12.23 MB/sec execute 210 sec latency 41.705 ms 2024-04-22T15:17:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:21 smithi186 ceph-mon[49804]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.9 MiB/s wr, 651 op/s 2024-04-22T15:17:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi186.vtjgwg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:17:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:17:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:22.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:22 smithi186 ceph-mon[49804]: Reconfiguring mgr.smithi186.vtjgwg (monmap changed)... 2024-04-22T15:17:22.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:22 smithi186 ceph-mon[49804]: Reconfiguring daemon mgr.smithi186.vtjgwg on smithi186 2024-04-22T15:17:22.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:22 smithi119 ceph-mon[63156]: Reconfiguring mgr.smithi186.vtjgwg (monmap changed)... 2024-04-22T15:17:22.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:22 smithi119 ceph-mon[63156]: Reconfiguring daemon mgr.smithi186.vtjgwg on smithi186 2024-04-22T15:17:22.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 170787 12.22 MB/sec execute 211 sec latency 44.738 ms 2024-04-22T15:17:23.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 171080 12.19 MB/sec execute 212 sec latency 46.398 ms 2024-04-22T15:17:23.829 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:23 smithi119 ceph-mon[63156]: pgmap v32: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 693 op/s 2024-04-22T15:17:23.829 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:23.830 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:23.830 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:17:23.830 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:17:23.830 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:24.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:23 smithi186 ceph-mon[49804]: pgmap v32: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 693 op/s 2024-04-22T15:17:24.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:24.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:24.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:17:24.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:17:24.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:24.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 171295 12.16 MB/sec execute 213 sec latency 45.162 ms 2024-04-22T15:17:24.853 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:24 smithi119 ceph-mon[63156]: Reconfiguring mon.smithi186 (monmap changed)... 2024-04-22T15:17:24.853 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:24 smithi119 ceph-mon[63156]: Reconfiguring daemon mon.smithi186 on smithi186 2024-04-22T15:17:24.853 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:24 smithi119 ceph-mon[63156]: pgmap v33: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.4 MiB/s wr, 617 op/s 2024-04-22T15:17:25.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:24 smithi186 ceph-mon[49804]: Reconfiguring mon.smithi186 (monmap changed)... 2024-04-22T15:17:25.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:24 smithi186 ceph-mon[49804]: Reconfiguring daemon mon.smithi186 on smithi186 2024-04-22T15:17:25.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:24 smithi186 ceph-mon[49804]: pgmap v33: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.4 MiB/s wr, 617 op/s 2024-04-22T15:17:25.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 171590 12.15 MB/sec execute 214 sec latency 66.150 ms 2024-04-22T15:17:25.870 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:17:25.871 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:25.871 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:25.871 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:25 smithi119 ceph-mon[63156]: Reconfiguring osd.4 (monmap changed)... 2024-04-22T15:17:25.871 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:17:25.871 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:25.871 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:25 smithi119 ceph-mon[63156]: Reconfiguring daemon osd.4 on smithi186 2024-04-22T15:17:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:17:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:25 smithi186 ceph-mon[49804]: Reconfiguring osd.4 (monmap changed)... 2024-04-22T15:17:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:17:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:25 smithi186 ceph-mon[49804]: Reconfiguring daemon osd.4 on smithi186 2024-04-22T15:17:26.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 171917 12.12 MB/sec execute 215 sec latency 79.115 ms 2024-04-22T15:17:26.892 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:26 smithi119 ceph-mon[63156]: pgmap v34: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 800 op/s 2024-04-22T15:17:27.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:26 smithi186 ceph-mon[49804]: pgmap v34: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 800 op/s 2024-04-22T15:17:27.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 172327 12.11 MB/sec execute 216 sec latency 45.517 ms 2024-04-22T15:17:28.415 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:28.415 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:28.415 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:28 smithi119 ceph-mon[63156]: Reconfiguring osd.5 (monmap changed)... 2024-04-22T15:17:28.416 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:17:28.416 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:28.416 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:28 smithi119 ceph-mon[63156]: Reconfiguring daemon osd.5 on smithi186 2024-04-22T15:17:28.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 172654 12.08 MB/sec execute 217 sec latency 50.671 ms 2024-04-22T15:17:28.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:28.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:28.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:28 smithi186 ceph-mon[49804]: Reconfiguring osd.5 (monmap changed)... 2024-04-22T15:17:28.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:17:28.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:28.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:28 smithi186 ceph-mon[49804]: Reconfiguring daemon osd.5 on smithi186 2024-04-22T15:17:29.431 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:29 smithi119 ceph-mon[63156]: pgmap v35: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.1 MiB/s wr, 400 op/s 2024-04-22T15:17:29.432 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:29 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:29.432 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:29 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:29.432 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:29 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:17:29.432 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:29 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:29.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 173033 12.07 MB/sec execute 218 sec latency 37.312 ms 2024-04-22T15:17:29.524 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:29 smithi186 ceph-mon[49804]: pgmap v35: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.1 MiB/s wr, 400 op/s 2024-04-22T15:17:29.524 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:29 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:29.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:29 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:29.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:29 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:17:29.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:29 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:30.265 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:30 smithi119 ceph-mon[63156]: Reconfiguring osd.6 (monmap changed)... 2024-04-22T15:17:30.265 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:30 smithi119 ceph-mon[63156]: Reconfiguring daemon osd.6 on smithi186 2024-04-22T15:17:30.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 173342 12.05 MB/sec execute 219 sec latency 50.055 ms 2024-04-22T15:17:30.524 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:30 smithi186 ceph-mon[49804]: Reconfiguring osd.6 (monmap changed)... 2024-04-22T15:17:30.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:30 smithi186 ceph-mon[49804]: Reconfiguring daemon osd.6 on smithi186 2024-04-22T15:17:31.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 173629 12.05 MB/sec execute 220 sec latency 74.506 ms 2024-04-22T15:17:31.524 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:31 smithi186 ceph-mon[49804]: pgmap v36: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 541 op/s 2024-04-22T15:17:31.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:31.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:31.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:17:31.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:31.530 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:31 smithi119 ceph-mon[63156]: pgmap v36: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 541 op/s 2024-04-22T15:17:31.530 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:31.530 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:31.530 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:17:31.530 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:32.327 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:32 smithi119 ceph-mon[63156]: Reconfiguring osd.7 (monmap changed)... 2024-04-22T15:17:32.327 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:32 smithi119 ceph-mon[63156]: Reconfiguring daemon osd.7 on smithi186 2024-04-22T15:17:32.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 173872 12.03 MB/sec execute 221 sec latency 51.950 ms 2024-04-22T15:17:32.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:32 smithi186 ceph-mon[49804]: Reconfiguring osd.7 (monmap changed)... 2024-04-22T15:17:32.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:32 smithi186 ceph-mon[49804]: Reconfiguring daemon osd.7 on smithi186 2024-04-22T15:17:33.389 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:33 smithi119 ceph-mon[63156]: pgmap v37: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.1 MiB/s wr, 562 op/s 2024-04-22T15:17:33.389 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:33 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:33.390 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:33 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:33.390 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:33 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi186.dglszf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T15:17:33.390 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:33 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:33.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 174147 12.04 MB/sec execute 222 sec latency 50.640 ms 2024-04-22T15:17:33.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:33 smithi186 ceph-mon[49804]: pgmap v37: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.1 MiB/s wr, 562 op/s 2024-04-22T15:17:33.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:33 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:33.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:33 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:33.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:33 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi186.dglszf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T15:17:33.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:33 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:34.400 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:34 smithi119 ceph-mon[63156]: Reconfiguring mds.foofs.smithi186.dglszf (monmap changed)... 2024-04-22T15:17:34.400 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:34 smithi119 ceph-mon[63156]: Reconfiguring daemon mds.foofs.smithi186.dglszf on smithi186 2024-04-22T15:17:34.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 174541 12.02 MB/sec execute 223 sec latency 84.276 ms 2024-04-22T15:17:34.524 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:34 smithi186 ceph-mon[49804]: Reconfiguring mds.foofs.smithi186.dglszf (monmap changed)... 2024-04-22T15:17:34.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:34 smithi186 ceph-mon[49804]: Reconfiguring daemon mds.foofs.smithi186.dglszf on smithi186 2024-04-22T15:17:35.413 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:35 smithi119 ceph-mon[63156]: pgmap v38: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 398 op/s 2024-04-22T15:17:35.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 174852 11.99 MB/sec execute 224 sec latency 37.041 ms 2024-04-22T15:17:35.667 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:35 smithi186 ceph-mon[49804]: pgmap v38: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 398 op/s 2024-04-22T15:17:36.438 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:36.439 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:36.439 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:17:36.439 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:17:36.439 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:17:36.439 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:36 smithi119 ceph-mon[63156]: Upgrade: Setting container_image for all mon 2024-04-22T15:17:36.439 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:36.439 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi119"}]: dispatch 2024-04-22T15:17:36.439 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi119"}]': finished 2024-04-22T15:17:36.439 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi186"}]: dispatch 2024-04-22T15:17:36.439 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi186"}]': finished 2024-04-22T15:17:36.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 175185 11.98 MB/sec execute 225 sec latency 33.133 ms 2024-04-22T15:17:36.659 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:36.660 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:36.660 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:17:36.660 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:17:36.660 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:17:36.660 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:36 smithi186 ceph-mon[49804]: Upgrade: Setting container_image for all mon 2024-04-22T15:17:36.660 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:36.660 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi119"}]: dispatch 2024-04-22T15:17:36.660 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi119"}]': finished 2024-04-22T15:17:36.660 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi186"}]: dispatch 2024-04-22T15:17:36.660 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi186"}]': finished 2024-04-22T15:17:37.455 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:37 smithi119 ceph-mon[63156]: pgmap v39: 97 pgs: 97 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.9 MiB/s wr, 589 op/s 2024-04-22T15:17:37.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 175539 11.96 MB/sec execute 226 sec latency 84.444 ms 2024-04-22T15:17:37.774 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:37 smithi186 ceph-mon[49804]: pgmap v39: 97 pgs: 97 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.9 MiB/s wr, 589 op/s 2024-04-22T15:17:38.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 175845 11.93 MB/sec execute 227 sec latency 53.941 ms 2024-04-22T15:17:39.004 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:38 smithi119 ceph-mon[63156]: Upgrade: Updating crash.smithi119 (1/2) 2024-04-22T15:17:39.005 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:39.005 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi119", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:17:39.005 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:39.005 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:38 smithi119 ceph-mon[63156]: Deploying daemon crash.smithi119 on smithi119 2024-04-22T15:17:39.005 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:38 smithi119 ceph-mon[63156]: pgmap v40: 97 pgs: 97 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-04-22T15:17:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:38 smithi186 ceph-mon[49804]: Upgrade: Updating crash.smithi119 (1/2) 2024-04-22T15:17:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:38 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:38 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi119", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:17:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:38 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:38 smithi186 ceph-mon[49804]: Deploying daemon crash.smithi119 on smithi119 2024-04-22T15:17:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:38 smithi186 ceph-mon[49804]: pgmap v40: 97 pgs: 97 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-04-22T15:17:39.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 176107 11.91 MB/sec execute 228 sec latency 59.147 ms 2024-04-22T15:17:40.022 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:17:40.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:39 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:17:40.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 176451 11.90 MB/sec execute 229 sec latency 50.434 ms 2024-04-22T15:17:41.061 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:40 smithi119 ceph-mon[63156]: pgmap v41: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.8 MiB/s wr, 446 op/s 2024-04-22T15:17:41.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:40 smithi186 ceph-mon[49804]: pgmap v41: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.8 MiB/s wr, 446 op/s 2024-04-22T15:17:41.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 176802 11.88 MB/sec execute 230 sec latency 47.544 ms 2024-04-22T15:17:42.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 177129 11.88 MB/sec execute 231 sec latency 46.559 ms 2024-04-22T15:17:43.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 177370 11.87 MB/sec execute 232 sec latency 46.004 ms 2024-04-22T15:17:43.854 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:43 smithi119 ceph-mon[63156]: pgmap v42: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.6 MiB/s wr, 465 op/s 2024-04-22T15:17:44.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:43 smithi186 ceph-mon[49804]: pgmap v42: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.6 MiB/s wr, 465 op/s 2024-04-22T15:17:44.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 177811 11.88 MB/sec execute 233 sec latency 42.098 ms 2024-04-22T15:17:44.862 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:44 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:44.863 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:44 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:45.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:44 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:45.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:44 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:45.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 178336 11.88 MB/sec execute 234 sec latency 25.084 ms 2024-04-22T15:17:45.888 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:45 smithi119 ceph-mon[63156]: pgmap v43: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 390 op/s 2024-04-22T15:17:45.888 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:45 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:45.889 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:45 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi186", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:17:45.889 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:45 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:46.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:45 smithi186 ceph-mon[49804]: pgmap v43: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 390 op/s 2024-04-22T15:17:46.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:45 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:46.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:45 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi186", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T15:17:46.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:45 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:46.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 178709 11.87 MB/sec execute 235 sec latency 40.557 ms 2024-04-22T15:17:46.901 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:46 smithi119 ceph-mon[63156]: Upgrade: Updating crash.smithi186 (2/2) 2024-04-22T15:17:46.901 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:46 smithi119 ceph-mon[63156]: Deploying daemon crash.smithi186 on smithi186 2024-04-22T15:17:47.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:46 smithi186 ceph-mon[49804]: Upgrade: Updating crash.smithi186 (2/2) 2024-04-22T15:17:47.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:46 smithi186 ceph-mon[49804]: Deploying daemon crash.smithi186 on smithi186 2024-04-22T15:17:47.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 179083 11.86 MB/sec execute 236 sec latency 45.016 ms 2024-04-22T15:17:47.806 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:47 smithi119 ceph-mon[63156]: pgmap v44: 97 pgs: 97 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.3 MiB/s wr, 618 op/s 2024-04-22T15:17:47.965 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:47 smithi186 ceph-mon[49804]: pgmap v44: 97 pgs: 97 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.3 MiB/s wr, 618 op/s 2024-04-22T15:17:48.486 INFO:teuthology.orchestra.run.smithi119.stdout: 5 179505 11.85 MB/sec execute 237 sec latency 42.477 ms 2024-04-22T15:17:49.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 179834 11.83 MB/sec execute 238 sec latency 48.389 ms 2024-04-22T15:17:49.848 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:49 smithi119 ceph-mon[63156]: pgmap v45: 97 pgs: 97 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 427 op/s 2024-04-22T15:17:49.848 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:49 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:49.848 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:49 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:50.002 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:17:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:49 smithi186 ceph-mon[49804]: pgmap v45: 97 pgs: 97 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 427 op/s 2024-04-22T15:17:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:49 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:49 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:50.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 180283 11.83 MB/sec execute 239 sec latency 40.395 ms 2024-04-22T15:17:50.854 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:50 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:17:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:50 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:17:51.148 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:17:51.148 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (3m) 76s ago 12m 15.8M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:17:51.148 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 starting - - - - 2024-04-22T15:17:51.148 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 starting - - - - 2024-04-22T15:17:51.148 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (3m) 76s ago 12m 45.4M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (6m) 76s ago 6m 38.3M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (6m) 61s ago 6m 14.0M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (2m) 76s ago 13m 479M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (118s) 61s ago 11m 449M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (98s) 76s ago 13m 33.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (67s) 61s ago 11m 25.5M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (6m) 76s ago 6m 270M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (6m) 61s ago 6m 84.3M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (3m) 76s ago 12m 12.4M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (3m) 61s ago 11m 16.7M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (10m) 76s ago 10m 508M 1936M 17.2.0 e1d6a67b021e 9999e9684eba 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (10m) 76s ago 10m 444M 1936M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (10m) 76s ago 10m 327M 1936M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (9m) 76s ago 9m 472M 1936M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:17:51.149 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (9m) 61s ago 9m 501M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:17:51.150 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (8m) 61s ago 8m 325M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:17:51.150 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (8m) 61s ago 8m 512M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:17:51.150 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (8m) 61s ago 8m 393M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:17:51.150 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (2m) 76s ago 11m 60.3M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:17:51.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 180711 11.84 MB/sec execute 240 sec latency 37.905 ms 2024-04-22T15:17:51.835 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:17:51.835 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:17:51.835 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:17:51.835 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:17:51.835 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:17:51.835 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:17:51.835 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:17:51.835 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:17:51.835 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:17:51.836 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:17:51.836 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:17:51.836 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:17:51.836 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:17:51.836 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:17:51.836 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-22T15:17:51.836 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 4 2024-04-22T15:17:51.836 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:17:51.836 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:17:51.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:51 smithi119 ceph-mon[63156]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:51.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:51 smithi119 ceph-mon[63156]: from='client.44127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:51.859 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:51 smithi119 ceph-mon[63156]: pgmap v46: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.5 MiB/s wr, 570 op/s 2024-04-22T15:17:51.865 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:51 smithi186 ceph-mon[49804]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:51.865 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:51 smithi186 ceph-mon[49804]: from='client.44127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:51.865 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:51 smithi186 ceph-mon[49804]: pgmap v46: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.5 MiB/s wr, 570 op/s 2024-04-22T15:17:52.467 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:17:52.467 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:17:52.467 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:17:52.467 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:17:52.467 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:17:52.468 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:17:52.468 INFO:teuthology.orchestra.run.smithi119.stdout: "mon" 2024-04-22T15:17:52.468 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:17:52.468 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "4/23 daemons upgraded", 2024-04-22T15:17:52.468 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading crash daemons", 2024-04-22T15:17:52.468 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:17:52.468 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:17:52.485 INFO:teuthology.orchestra.run.smithi119.stdout: 5 181070 11.85 MB/sec execute 241 sec latency 49.013 ms 2024-04-22T15:17:52.867 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:52 smithi119 ceph-mon[63156]: from='client.44131 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:52.867 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:52 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/1454781767' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:17:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:52 smithi186 ceph-mon[49804]: from='client.44131 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:52 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/1454781767' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:17:53.157 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:17:53.486 INFO:teuthology.orchestra.run.smithi119.stdout: 5 181531 11.85 MB/sec execute 242 sec latency 40.536 ms 2024-04-22T15:17:53.882 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:53 smithi119 ceph-mon[63156]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:53.882 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:53 smithi119 ceph-mon[63156]: pgmap v47: 97 pgs: 97 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 628 op/s 2024-04-22T15:17:53.882 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:53 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/586471776' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:17:54.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:53 smithi186 ceph-mon[49804]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:17:54.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:53 smithi186 ceph-mon[49804]: pgmap v47: 97 pgs: 97 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 628 op/s 2024-04-22T15:17:54.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:53 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/586471776' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:17:54.486 INFO:teuthology.orchestra.run.smithi119.stdout: 5 181968 11.84 MB/sec execute 243 sec latency 28.920 ms 2024-04-22T15:17:54.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:54 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:54.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:54 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:54.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:54 smithi119 ceph-mon[63156]: pgmap v48: 97 pgs: 97 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 468 op/s 2024-04-22T15:17:54.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:54 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:17:55.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:54 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:55.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:54 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:55.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:54 smithi186 ceph-mon[49804]: pgmap v48: 97 pgs: 97 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 468 op/s 2024-04-22T15:17:55.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:54 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:17:55.486 INFO:teuthology.orchestra.run.smithi119.stdout: 5 182365 11.83 MB/sec execute 244 sec latency 33.948 ms 2024-04-22T15:17:56.417 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:56 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:56.417 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:56 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:56.486 INFO:teuthology.orchestra.run.smithi119.stdout: 5 182834 11.82 MB/sec execute 245 sec latency 48.490 ms 2024-04-22T15:17:56.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:56 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:56.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:56 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:17:57.221 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:57 smithi119 ceph-mon[63156]: pgmap v49: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.5 MiB/s wr, 734 op/s 2024-04-22T15:17:57.486 INFO:teuthology.orchestra.run.smithi119.stdout: 5 183264 11.82 MB/sec execute 246 sec latency 51.323 ms 2024-04-22T15:17:57.524 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:57 smithi186 ceph-mon[49804]: pgmap v49: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.5 MiB/s wr, 734 op/s 2024-04-22T15:17:58.226 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:58 smithi119 ceph-mon[63156]: mgrmap e42: smithi119.jjbxxd(active, since 92s), standbys: smithi186.vtjgwg 2024-04-22T15:17:58.487 INFO:teuthology.orchestra.run.smithi119.stdout: 5 183904 11.84 MB/sec execute 247 sec latency 50.637 ms 2024-04-22T15:17:58.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:58 smithi186 ceph-mon[49804]: mgrmap e42: smithi119.jjbxxd(active, since 92s), standbys: smithi186.vtjgwg 2024-04-22T15:17:59.238 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:17:59 smithi119 ceph-mon[63156]: pgmap v50: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 505 op/s 2024-04-22T15:17:59.366 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:17:59 smithi186 ceph-mon[49804]: pgmap v50: 97 pgs: 97 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 505 op/s 2024-04-22T15:17:59.487 INFO:teuthology.orchestra.run.smithi119.stdout: 5 184531 11.88 MB/sec execute 248 sec latency 23.649 ms 2024-04-22T15:18:00.487 INFO:teuthology.orchestra.run.smithi119.stdout: 5 185309 11.92 MB/sec execute 249 sec latency 42.752 ms 2024-04-22T15:18:00.496 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:00 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:00.496 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:00 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:00.496 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:00 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:00.497 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:00 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:00 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:00 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:00 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:00 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:01.488 INFO:teuthology.orchestra.run.smithi119.stdout: 5 185587 11.89 MB/sec execute 250 sec latency 75.385 ms 2024-04-22T15:18:01.511 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:01 smithi119 ceph-mon[63156]: pgmap v51: 97 pgs: 97 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.6 MiB/s wr, 697 op/s 2024-04-22T15:18:01.774 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:01 smithi186 ceph-mon[49804]: pgmap v51: 97 pgs: 97 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.6 MiB/s wr, 697 op/s 2024-04-22T15:18:02.487 INFO:teuthology.orchestra.run.smithi119.stdout: 5 185851 11.88 MB/sec execute 251 sec latency 74.855 ms 2024-04-22T15:18:02.524 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:02 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:02.524 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:02 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:02.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:02 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:02.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:02 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:03.487 INFO:teuthology.orchestra.run.smithi119.stdout: 5 186172 11.86 MB/sec execute 252 sec latency 55.121 ms 2024-04-22T15:18:03.564 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:03 smithi119 ceph-mon[63156]: pgmap v52: 97 pgs: 97 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 714 op/s 2024-04-22T15:18:03.774 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:03 smithi186 ceph-mon[49804]: pgmap v52: 97 pgs: 97 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 714 op/s 2024-04-22T15:18:04.488 INFO:teuthology.orchestra.run.smithi119.stdout: 5 186472 11.84 MB/sec execute 253 sec latency 61.364 ms 2024-04-22T15:18:05.488 INFO:teuthology.orchestra.run.smithi119.stdout: 5 187066 11.86 MB/sec execute 254 sec latency 33.224 ms 2024-04-22T15:18:05.841 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:05 smithi119 ceph-mon[63156]: pgmap v53: 97 pgs: 97 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 615 op/s 2024-04-22T15:18:06.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:05 smithi186 ceph-mon[49804]: pgmap v53: 97 pgs: 97 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 615 op/s 2024-04-22T15:18:06.488 INFO:teuthology.orchestra.run.smithi119.stdout: 5 187659 11.88 MB/sec execute 255 sec latency 29.251 ms 2024-04-22T15:18:07.488 INFO:teuthology.orchestra.run.smithi119.stdout: 5 188013 11.87 MB/sec execute 256 sec latency 55.179 ms 2024-04-22T15:18:07.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: pgmap v54: 97 pgs: 97 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 836 op/s 2024-04-22T15:18:07.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:07.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:07.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:07.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:18:07.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:07.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:07.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:07.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:07.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:07.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:07.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:07.907 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi119"}]: dispatch 2024-04-22T15:18:07.907 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi119"}]': finished 2024-04-22T15:18:07.907 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi186"}]: dispatch 2024-04-22T15:18:07.907 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi186"}]': finished 2024-04-22T15:18:07.907 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:07 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T15:18:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: pgmap v54: 97 pgs: 97 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 836 op/s 2024-04-22T15:18:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi119"}]: dispatch 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi119"}]': finished 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi186"}]: dispatch 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi186"}]': finished 2024-04-22T15:18:08.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:07 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T15:18:08.488 INFO:teuthology.orchestra.run.smithi119.stdout: 5 188337 11.87 MB/sec execute 257 sec latency 45.077 ms 2024-04-22T15:18:08.915 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:08 smithi119 ceph-mon[63156]: Upgrade: Setting container_image for all crash 2024-04-22T15:18:08.915 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:08 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T15:18:08.916 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:08 smithi119 ceph-mon[63156]: Upgrade: osd.0 is safe to restart 2024-04-22T15:18:09.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:08 smithi186 ceph-mon[49804]: Upgrade: Setting container_image for all crash 2024-04-22T15:18:09.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:08 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T15:18:09.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:08 smithi186 ceph-mon[49804]: Upgrade: osd.0 is safe to restart 2024-04-22T15:18:09.488 INFO:teuthology.orchestra.run.smithi119.stdout: 5 188659 11.85 MB/sec execute 258 sec latency 44.158 ms 2024-04-22T15:18:09.934 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:09 smithi119 ceph-mon[63156]: pgmap v55: 97 pgs: 97 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.7 MiB/s wr, 571 op/s 2024-04-22T15:18:10.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:09 smithi186 ceph-mon[49804]: pgmap v55: 97 pgs: 97 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.7 MiB/s wr, 571 op/s 2024-04-22T15:18:10.488 INFO:teuthology.orchestra.run.smithi119.stdout: 5 189073 11.85 MB/sec execute 259 sec latency 68.418 ms 2024-04-22T15:18:10.941 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:10.941 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:10.942 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:18:10.942 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:11.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:18:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:11.488 INFO:teuthology.orchestra.run.smithi119.stdout: 5 189544 11.85 MB/sec execute 260 sec latency 42.586 ms 2024-04-22T15:18:11.948 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:11 smithi119 ceph-mon[63156]: Upgrade: Updating osd.0 2024-04-22T15:18:11.948 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:11 smithi119 ceph-mon[63156]: Deploying daemon osd.0 on smithi119 2024-04-22T15:18:11.948 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:11 smithi119 ceph-mon[63156]: pgmap v56: 97 pgs: 97 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.6 MiB/s wr, 710 op/s 2024-04-22T15:18:12.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:11 smithi186 ceph-mon[49804]: Upgrade: Updating osd.0 2024-04-22T15:18:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:11 smithi186 ceph-mon[49804]: Deploying daemon osd.0 on smithi119 2024-04-22T15:18:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:11 smithi186 ceph-mon[49804]: pgmap v56: 97 pgs: 97 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.6 MiB/s wr, 710 op/s 2024-04-22T15:18:12.488 INFO:teuthology.orchestra.run.smithi119.stdout: 5 190082 11.86 MB/sec execute 261 sec latency 45.317 ms 2024-04-22T15:18:13.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 190524 11.85 MB/sec execute 262 sec latency 36.811 ms 2024-04-22T15:18:13.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:13 smithi119 ceph-mon[63156]: pgmap v57: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 5.6 MiB/s wr, 622 op/s 2024-04-22T15:18:14.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:13 smithi186 ceph-mon[49804]: pgmap v57: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 5.6 MiB/s wr, 622 op/s 2024-04-22T15:18:14.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 190935 11.86 MB/sec execute 263 sec latency 62.127 ms 2024-04-22T15:18:15.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 191421 11.87 MB/sec execute 264 sec latency 33.746 ms 2024-04-22T15:18:15.775 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:15 smithi119 ceph-mon[63156]: pgmap v58: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.6 MiB/s wr, 463 op/s 2024-04-22T15:18:16.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:15 smithi186 ceph-mon[49804]: pgmap v58: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.6 MiB/s wr, 463 op/s 2024-04-22T15:18:16.030 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:15 smithi119 systemd[1]: Stopping Ceph osd.0 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:18:16.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 191832 11.87 MB/sec execute 265 sec latency 26.540 ms 2024-04-22T15:18:16.535 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:16 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0[32253]: 2024-04-22T15:18:16.321+0000 7fab47def700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:18:16.536 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:16 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0[32253]: 2024-04-22T15:18:16.321+0000 7fab47def700 -1 osd.0 72 *** Got signal Terminated *** 2024-04-22T15:18:16.536 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:16 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0[32253]: 2024-04-22T15:18:16.321+0000 7fab47def700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:18:16.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:16 smithi119 ceph-mon[63156]: osd.0 marked itself down and dead 2024-04-22T15:18:17.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:16 smithi186 ceph-mon[49804]: osd.0 marked itself down and dead 2024-04-22T15:18:17.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 191965 11.84 MB/sec execute 266 sec latency 715.648 ms 2024-04-22T15:18:17.714 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:17 smithi119 podman[68672]: 2024-04-22 15:18:17.439105416 +0000 UTC m=+1.283708995 container died 9999e9684ebaec4753ccf33250661a89c0ac8fd8f76bbdde1fd4af41e68575f3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, ceph=True, vendor=Red Hat, Inc., vcs-type=git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_CLEAN=True, name=centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, release=754) 2024-04-22T15:18:17.968 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:17 smithi119 ceph-mon[63156]: pgmap v59: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 747 op/s 2024-04-22T15:18:17.969 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:17 smithi119 ceph-mon[63156]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:18:17.969 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:17 smithi119 ceph-mon[63156]: osdmap e73: 8 total, 7 up, 8 in 2024-04-22T15:18:17.969 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:17 smithi119 podman[68672]: 2024-04-22 15:18:17.811028665 +0000 UTC m=+1.655632233 container cleanup 9999e9684ebaec4753ccf33250661a89c0ac8fd8f76bbdde1fd4af41e68575f3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_CLEAN=True, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, ceph=True, vcs-type=git, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-22T15:18:17.969 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:17 smithi119 bash[68672]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0 2024-04-22T15:18:18.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:17 smithi186 ceph-mon[49804]: pgmap v59: 97 pgs: 97 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 747 op/s 2024-04-22T15:18:18.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:17 smithi186 ceph-mon[49804]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:18:18.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:17 smithi186 ceph-mon[49804]: osdmap e73: 8 total, 7 up, 8 in 2024-04-22T15:18:18.221 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:18 smithi119 podman[68686]: 2024-04-22 15:18:18.188155796 +0000 UTC m=+0.748637629 container remove 9999e9684ebaec4753ccf33250661a89c0ac8fd8f76bbdde1fd4af41e68575f3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0, RELEASE=HEAD, io.openshift.expose-services=, distribution-scope=public, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , name=centos-stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vendor=Red Hat, Inc., version=8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True) 2024-04-22T15:18:18.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 192184 11.81 MB/sec execute 267 sec latency 1064.916 ms 2024-04-22T15:18:18.733 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:18 smithi119 podman[68811]: 2024-04-22 15:18:18.48692938 +0000 UTC m=+0.020405315 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:18:18.733 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:18 smithi119 podman[68811]: 2024-04-22 15:18:18.64764396 +0000 UTC m=+0.181119886 container create dd3b79aeec0550f625723a97e49b2e241924456f1f5e6a7bb5d136453a4b5c52 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-22T15:18:18.985 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:18 smithi119 podman[68811]: 2024-04-22 15:18:18.900847372 +0000 UTC m=+0.434323294 container init dd3b79aeec0550f625723a97e49b2e241924456f1f5e6a7bb5d136453a4b5c52 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416) 2024-04-22T15:18:18.985 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:18 smithi119 podman[68811]: 2024-04-22 15:18:18.904670891 +0000 UTC m=+0.438146819 container start dd3b79aeec0550f625723a97e49b2e241924456f1f5e6a7bb5d136453a4b5c52 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:18:18.988 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:18 smithi119 ceph-mon[63156]: osdmap e74: 8 total, 7 up, 8 in 2024-04-22T15:18:18.988 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:18 smithi119 ceph-mon[63156]: pgmap v62: 97 pgs: 13 stale+active+clean, 84 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 581 op/s 2024-04-22T15:18:19.241 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:18 smithi119 podman[68811]: 2024-04-22 15:18:18.998095621 +0000 UTC m=+0.531571549 container attach dd3b79aeec0550f625723a97e49b2e241924456f1f5e6a7bb5d136453a4b5c52 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240416, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-22T15:18:19.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:18 smithi186 ceph-mon[49804]: osdmap e74: 8 total, 7 up, 8 in 2024-04-22T15:18:19.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:18 smithi186 ceph-mon[49804]: pgmap v62: 97 pgs: 13 stale+active+clean, 84 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 581 op/s 2024-04-22T15:18:19.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 192418 11.78 MB/sec execute 268 sec latency 80.245 ms 2024-04-22T15:18:19.516 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:19 smithi119 podman[68811]: 2024-04-22 15:18:19.333830497 +0000 UTC m=+0.867306417 container died dd3b79aeec0550f625723a97e49b2e241924456f1f5e6a7bb5d136453a4b5c52 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-22T15:18:20.311 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:20 smithi119 podman[68837]: 2024-04-22 15:18:20.193513523 +0000 UTC m=+0.854362819 container remove dd3b79aeec0550f625723a97e49b2e241924456f1f5e6a7bb5d136453a4b5c52 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-22T15:18:20.311 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:20 smithi119 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-22T15:18:20.311 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:20 smithi119 systemd[1]: Stopped Ceph osd.0 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:18:20.311 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:20 smithi119 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.0.service: Consumed 3min 18.005s CPU time. 2024-04-22T15:18:20.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 192645 11.77 MB/sec execute 269 sec latency 77.716 ms 2024-04-22T15:18:20.562 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:20 smithi119 systemd[1]: Starting Ceph osd.0 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:18:20.813 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:20 smithi119 ceph-mon[63156]: pgmap v63: 97 pgs: 14 active+undersized+degraded, 2 active+undersized, 74 active+clean, 7 stale+active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.0 MiB/s wr, 457 op/s; 105/1695 objects degraded (6.195%) 2024-04-22T15:18:20.814 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:20 smithi119 podman[68933]: 2024-04-22 15:18:20.680194807 +0000 UTC m=+0.021507319 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:18:21.064 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:20 smithi119 ceph-mon[63156]: Health check failed: Degraded data redundancy: 105/1695 objects degraded (6.195%), 14 pgs degraded (PG_DEGRADED) 2024-04-22T15:18:21.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:20 smithi186 ceph-mon[49804]: pgmap v63: 97 pgs: 14 active+undersized+degraded, 2 active+undersized, 74 active+clean, 7 stale+active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.0 MiB/s wr, 457 op/s; 105/1695 objects degraded (6.195%) 2024-04-22T15:18:21.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:20 smithi186 ceph-mon[49804]: Health check failed: Degraded data redundancy: 105/1695 objects degraded (6.195%), 14 pgs degraded (PG_DEGRADED) 2024-04-22T15:18:21.359 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:21 smithi119 podman[68933]: 2024-04-22 15:18:21.10311454 +0000 UTC m=+0.444427043 container create 41d2ced7c1287aabf68dee09fe087e035586115ad0da497786fd96c40563632e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2) 2024-04-22T15:18:21.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 192836 11.74 MB/sec execute 270 sec latency 102.498 ms 2024-04-22T15:18:21.610 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:21 smithi119 podman[68933]: 2024-04-22 15:18:21.590584624 +0000 UTC m=+0.931897127 container init 41d2ced7c1287aabf68dee09fe087e035586115ad0da497786fd96c40563632e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS) 2024-04-22T15:18:21.610 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:21 smithi119 podman[68933]: 2024-04-22 15:18:21.602325194 +0000 UTC m=+0.943637708 container start 41d2ced7c1287aabf68dee09fe087e035586115ad0da497786fd96c40563632e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-04-22T15:18:21.861 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:21 smithi119 podman[68933]: 2024-04-22 15:18:21.749857181 +0000 UTC m=+1.091169687 container attach 41d2ced7c1287aabf68dee09fe087e035586115ad0da497786fd96c40563632e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True) 2024-04-22T15:18:22.378 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate[68947]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:18:22.378 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate[68947]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:18:22.378 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate[68947]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T15:18:22.378 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 bash[68933]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:18:22.378 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 bash[68933]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:18:22.378 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 bash[68933]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T15:18:22.378 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 bash[68933]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T15:18:22.378 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 bash[68933]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T15:18:22.378 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 bash[68933]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T15:18:22.378 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 bash[68933]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-22T15:18:22.379 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 bash[68933]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T15:18:22.379 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 bash[68933]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-22T15:18:22.379 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate[68947]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T15:18:22.379 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate[68947]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T15:18:22.379 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate[68947]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T15:18:22.379 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate[68947]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-22T15:18:22.379 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate[68947]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T15:18:22.379 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate[68947]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-22T15:18:22.379 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:22 smithi119 podman[68933]: 2024-04-22 15:18:22.246113887 +0000 UTC m=+1.587426397 container died 41d2ced7c1287aabf68dee09fe087e035586115ad0da497786fd96c40563632e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:18:22.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 193177 11.73 MB/sec execute 271 sec latency 53.674 ms 2024-04-22T15:18:23.415 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:23 smithi119 podman[69090]: 2024-04-22 15:18:23.182163881 +0000 UTC m=+0.930095464 container remove 41d2ced7c1287aabf68dee09fe087e035586115ad0da497786fd96c40563632e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=) 2024-04-22T15:18:23.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 193504 11.70 MB/sec execute 272 sec latency 63.013 ms 2024-04-22T15:18:23.668 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:23 smithi119 podman[69165]: 2024-04-22 15:18:23.397471131 +0000 UTC m=+0.029611487 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:18:23.668 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:23 smithi119 podman[69165]: 2024-04-22 15:18:23.593762988 +0000 UTC m=+0.225903337 container create b35da6e9f2fab056c52fa0b6ba1ee00d77f28e5d0d66e8ebd6de429778a1300f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:18:23.729 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:18:23.919 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:23 smithi119 ceph-mon[63156]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.6 MiB/s wr, 655 op/s; 224/1644 objects degraded (13.625%); 928 B/s, 0 objects/s recovering 2024-04-22T15:18:23.919 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:23 smithi119 podman[69165]: 2024-04-22 15:18:23.905065845 +0000 UTC m=+0.537206202 container init b35da6e9f2fab056c52fa0b6ba1ee00d77f28e5d0d66e8ebd6de429778a1300f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:18:23.920 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:23 smithi119 podman[69165]: 2024-04-22 15:18:23.916297024 +0000 UTC m=+0.548437374 container start b35da6e9f2fab056c52fa0b6ba1ee00d77f28e5d0d66e8ebd6de429778a1300f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS) 2024-04-22T15:18:24.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:23 smithi186 ceph-mon[49804]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.6 MiB/s wr, 655 op/s; 224/1644 objects degraded (13.625%); 928 B/s, 0 objects/s recovering 2024-04-22T15:18:24.170 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:24 smithi119 bash[69165]: b35da6e9f2fab056c52fa0b6ba1ee00d77f28e5d0d66e8ebd6de429778a1300f 2024-04-22T15:18:24.170 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:24 smithi119 systemd[1]: Started Ceph osd.0 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:18:24.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 193854 11.71 MB/sec execute 273 sec latency 56.110 ms 2024-04-22T15:18:24.674 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:24 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0[69183]: 2024-04-22T15:18:24.499+0000 7fd98fe9b7c0 -1 Falling back to public interface 2024-04-22T15:18:24.853 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:18:24.853 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (4m) 24s ago 13m 15.8M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:18:24.853 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (41s) 24s ago 12m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:18:24.853 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (35s) 30s ago 12m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:18:24.853 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (3m) 24s ago 12m 47.4M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:18:24.853 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (7m) 24s ago 7m 38.5M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:18:24.853 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (7m) 30s ago 7m 14.2M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:18:24.853 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (3m) 24s ago 14m 499M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:18:24.853 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (2m) 30s ago 12m 450M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:18:24.853 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (2m) 24s ago 14m 45.3M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:18:24.853 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (101s) 30s ago 12m 38.9M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (6m) 24s ago 6m 271M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (6m) 30s ago 6m 84.4M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (3m) 24s ago 12m 10.9M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (3m) 30s ago 12m 17.4M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 starting - - - 1936M 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (11m) 24s ago 11m 555M 1936M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (10m) 24s ago 10m 422M 1936M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (10m) 24s ago 10m 540M 1936M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (9m) 30s ago 9m 545M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (9m) 30s ago 9m 409M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (9m) 30s ago 9m 581M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (8m) 30s ago 8m 483M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:18:24.854 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (2m) 24s ago 11m 63.0M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:18:24.928 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:24 smithi119 ceph-mon[63156]: from='client.44147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:24.928 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:24.928 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:24.928 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:25.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:24 smithi186 ceph-mon[49804]: from='client.44147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:25.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:24 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:25.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:24 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:25.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:24 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:25.489 INFO:teuthology.orchestra.run.smithi119.stdout: 5 194106 11.68 MB/sec execute 274 sec latency 46.411 ms 2024-04-22T15:18:25.510 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:18:25.510 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:18:25.510 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:18:25.510 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:18:25.510 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 4 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:18:25.511 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:18:25.942 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:25 smithi119 ceph-mon[63156]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:25.943 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:25 smithi119 ceph-mon[63156]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.3 MiB/s wr, 230 op/s; 224/1644 objects degraded (13.625%); 928 B/s, 0 objects/s recovering 2024-04-22T15:18:25.943 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:25.943 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:25.943 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:25 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/2025091415' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:26.013 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:18:26.013 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:18:26.013 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:18:26.013 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:18:26.013 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:18:26.013 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:18:26.013 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:18:26.014 INFO:teuthology.orchestra.run.smithi119.stdout: "crash" 2024-04-22T15:18:26.014 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:18:26.014 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "6/23 daemons upgraded", 2024-04-22T15:18:26.014 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:18:26.014 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:18:26.014 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:18:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:25 smithi186 ceph-mon[49804]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:25 smithi186 ceph-mon[49804]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.3 MiB/s wr, 230 op/s; 224/1644 objects degraded (13.625%); 928 B/s, 0 objects/s recovering 2024-04-22T15:18:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:25 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/2025091415' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:26.490 INFO:teuthology.orchestra.run.smithi119.stdout: 5 194427 11.69 MB/sec execute 275 sec latency 44.279 ms 2024-04-22T15:18:26.648 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 224/1644 objects degraded (13.625%), 29 pgs degraded 2024-04-22T15:18:26.648 INFO:teuthology.orchestra.run.smithi119.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-22T15:18:26.648 INFO:teuthology.orchestra.run.smithi119.stdout: osd.0 (root=default,host=smithi119) is down 2024-04-22T15:18:26.649 INFO:teuthology.orchestra.run.smithi119.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 224/1644 objects degraded (13.625%), 29 pgs degraded 2024-04-22T15:18:26.649 INFO:teuthology.orchestra.run.smithi119.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-04-22T15:18:26.649 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-04-22T15:18:26.649 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-04-22T15:18:26.649 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-04-22T15:18:26.649 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-04-22T15:18:26.649 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-04-22T15:18:26.649 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-04-22T15:18:26.649 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.f is active+undersized+degraded, acting [4,2] 2024-04-22T15:18:26.650 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-04-22T15:18:26.650 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-04-22T15:18:26.650 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-04-22T15:18:26.650 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-04-22T15:18:26.650 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-04-22T15:18:26.650 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-04-22T15:18:26.650 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-04-22T15:18:26.650 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-04-22T15:18:26.650 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-04-22T15:18:26.650 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-04-22T15:18:26.650 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-04-22T15:18:26.650 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-04-22T15:18:26.651 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-04-22T15:18:26.651 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-04-22T15:18:26.651 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-04-22T15:18:26.651 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-04-22T15:18:26.651 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-04-22T15:18:26.651 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-04-22T15:18:26.651 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-04-22T15:18:26.652 INFO:teuthology.orchestra.run.smithi119.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-04-22T15:18:26.652 INFO:teuthology.orchestra.run.smithi119.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-04-22T15:18:26.948 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:26 smithi119 ceph-mon[63156]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:26.949 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:26 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/2269584541' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:18:27.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:26 smithi186 ceph-mon[49804]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:27.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:26 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/2269584541' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:18:27.490 INFO:teuthology.orchestra.run.smithi119.stdout: 5 194907 11.68 MB/sec execute 276 sec latency 71.772 ms 2024-04-22T15:18:27.960 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:27 smithi119 ceph-mon[63156]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:27.960 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:27 smithi119 ceph-mon[63156]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 330 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.0 MiB/s wr, 411 op/s; 219/1596 objects degraded (13.722%); 749 B/s, 0 objects/s recovering 2024-04-22T15:18:27.960 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:27 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 219/1596 objects degraded (13.722%), 29 pgs degraded (PG_DEGRADED) 2024-04-22T15:18:28.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:27 smithi186 ceph-mon[49804]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:28.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:27 smithi186 ceph-mon[49804]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 330 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.0 MiB/s wr, 411 op/s; 219/1596 objects degraded (13.722%); 749 B/s, 0 objects/s recovering 2024-04-22T15:18:28.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:27 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 219/1596 objects degraded (13.722%), 29 pgs degraded (PG_DEGRADED) 2024-04-22T15:18:28.490 INFO:teuthology.orchestra.run.smithi119.stdout: 5 195290 11.68 MB/sec execute 277 sec latency 29.367 ms 2024-04-22T15:18:29.490 INFO:teuthology.orchestra.run.smithi119.stdout: 5 195638 11.68 MB/sec execute 278 sec latency 53.055 ms 2024-04-22T15:18:29.758 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:29 smithi119 ceph-mon[63156]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 330 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 373 op/s; 219/1596 objects degraded (13.722%); 680 B/s, 0 objects/s recovering 2024-04-22T15:18:30.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:29 smithi186 ceph-mon[49804]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 330 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 373 op/s; 219/1596 objects degraded (13.722%); 680 B/s, 0 objects/s recovering 2024-04-22T15:18:30.490 INFO:teuthology.orchestra.run.smithi119.stdout: 5 195979 11.66 MB/sec execute 279 sec latency 55.335 ms 2024-04-22T15:18:31.490 INFO:teuthology.orchestra.run.smithi119.stdout: 5 196275 11.65 MB/sec execute 280 sec latency 78.149 ms 2024-04-22T15:18:31.798 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:31 smithi119 ceph-mon[63156]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 419 op/s; 222/1647 objects degraded (13.479%); 619 B/s, 0 objects/s recovering 2024-04-22T15:18:31.799 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:31 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0[69183]: 2024-04-22T15:18:31.588+0000 7fd98fe9b7c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-22T15:18:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:31 smithi186 ceph-mon[49804]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 419 op/s; 222/1647 objects degraded (13.479%); 619 B/s, 0 objects/s recovering 2024-04-22T15:18:32.490 INFO:teuthology.orchestra.run.smithi119.stdout: 5 196649 11.64 MB/sec execute 281 sec latency 51.272 ms 2024-04-22T15:18:32.828 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:32 smithi119 ceph-mon[63156]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 338 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.2 MiB/s wr, 543 op/s; 236/1728 objects degraded (13.657%); 619 B/s, 0 objects/s recovering 2024-04-22T15:18:32.829 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:32 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 236/1728 objects degraded (13.657%), 29 pgs degraded (PG_DEGRADED) 2024-04-22T15:18:33.079 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:32 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0[69183]: 2024-04-22T15:18:32.899+0000 7fd98fe9b7c0 -1 osd.0 72 log_to_monitors true 2024-04-22T15:18:33.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:32 smithi186 ceph-mon[49804]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 338 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.2 MiB/s wr, 543 op/s; 236/1728 objects degraded (13.657%); 619 B/s, 0 objects/s recovering 2024-04-22T15:18:33.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:32 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 236/1728 objects degraded (13.657%), 29 pgs degraded (PG_DEGRADED) 2024-04-22T15:18:33.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 197129 11.65 MB/sec execute 282 sec latency 35.630 ms 2024-04-22T15:18:33.848 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:33 smithi119 ceph-mon[63156]: from='osd.0 [v2:172.21.15.119:6802/545384490,v1:172.21.15.119:6803/545384490]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T15:18:34.099 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:18:33 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0[69183]: 2024-04-22T15:18:33.848+0000 7fd9812f3700 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:18:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:33 smithi186 ceph-mon[49804]: from='osd.0 [v2:172.21.15.119:6802/545384490,v1:172.21.15.119:6803/545384490]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T15:18:34.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 197547 11.64 MB/sec execute 283 sec latency 35.277 ms 2024-04-22T15:18:34.863 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:34 smithi119 ceph-mon[63156]: from='osd.0 [v2:172.21.15.119:6802/545384490,v1:172.21.15.119:6803/545384490]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T15:18:34.864 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:34 smithi119 ceph-mon[63156]: osdmap e75: 8 total, 7 up, 8 in 2024-04-22T15:18:34.864 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:34 smithi119 ceph-mon[63156]: from='osd.0 [v2:172.21.15.119:6802/545384490,v1:172.21.15.119:6803/545384490]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:18:34.864 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:34 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:34.864 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:34 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:34.864 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:34 smithi119 ceph-mon[63156]: pgmap v71: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 493 op/s; 236/1728 objects degraded (13.657%) 2024-04-22T15:18:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:34 smithi186 ceph-mon[49804]: from='osd.0 [v2:172.21.15.119:6802/545384490,v1:172.21.15.119:6803/545384490]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T15:18:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:34 smithi186 ceph-mon[49804]: osdmap e75: 8 total, 7 up, 8 in 2024-04-22T15:18:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:34 smithi186 ceph-mon[49804]: from='osd.0 [v2:172.21.15.119:6802/545384490,v1:172.21.15.119:6803/545384490]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:18:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:34 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:34 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:34 smithi186 ceph-mon[49804]: pgmap v71: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 493 op/s; 236/1728 objects degraded (13.657%) 2024-04-22T15:18:35.495 INFO:teuthology.orchestra.run.smithi119.stdout: 5 197755 11.62 MB/sec execute 284 sec latency 678.364 ms 2024-04-22T15:18:36.087 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:35 smithi119 ceph-mon[63156]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:18:36.088 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:35 smithi119 ceph-mon[63156]: osd.0 [v2:172.21.15.119:6802/545384490,v1:172.21.15.119:6803/545384490] boot 2024-04-22T15:18:36.088 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:35 smithi119 ceph-mon[63156]: osdmap e76: 8 total, 8 up, 8 in 2024-04-22T15:18:36.088 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:18:36.088 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:36.088 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:36.088 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:35 smithi119 ceph-mon[63156]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T15:18:36.176 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:35 smithi186 ceph-mon[49804]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:18:36.176 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:35 smithi186 ceph-mon[49804]: osd.0 [v2:172.21.15.119:6802/545384490,v1:172.21.15.119:6803/545384490] boot 2024-04-22T15:18:36.176 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:35 smithi186 ceph-mon[49804]: osdmap e76: 8 total, 8 up, 8 in 2024-04-22T15:18:36.176 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:18:36.176 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:36.176 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:36.176 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:35 smithi186 ceph-mon[49804]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T15:18:36.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 197881 11.61 MB/sec execute 285 sec latency 1678.388 ms 2024-04-22T15:18:36.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:36 smithi119 ceph-mon[63156]: pgmap v74: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.8 MiB/s wr, 671 op/s; 95/1704 objects degraded (5.575%) 2024-04-22T15:18:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:36 smithi186 ceph-mon[49804]: pgmap v74: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.8 MiB/s wr, 671 op/s; 95/1704 objects degraded (5.575%) 2024-04-22T15:18:37.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 197937 11.57 MB/sec execute 286 sec latency 2535.755 ms 2024-04-22T15:18:37.936 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:37 smithi119 ceph-mon[63156]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T15:18:38.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:37 smithi186 ceph-mon[49804]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T15:18:38.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 198348 11.56 MB/sec execute 287 sec latency 38.747 ms 2024-04-22T15:18:38.943 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:38 smithi119 ceph-mon[63156]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T15:18:38.943 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:38 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 95/1704 objects degraded (5.575%), 8 pgs degraded (PG_DEGRADED) 2024-04-22T15:18:38.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:38 smithi119 ceph-mon[63156]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T15:18:38.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:38 smithi119 ceph-mon[63156]: pgmap v78: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 591 op/s; 95/1704 objects degraded (5.575%) 2024-04-22T15:18:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:38 smithi186 ceph-mon[49804]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T15:18:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:38 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 95/1704 objects degraded (5.575%), 8 pgs degraded (PG_DEGRADED) 2024-04-22T15:18:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:38 smithi186 ceph-mon[49804]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T15:18:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:38 smithi186 ceph-mon[49804]: pgmap v78: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 591 op/s; 95/1704 objects degraded (5.575%) 2024-04-22T15:18:39.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 198595 11.54 MB/sec execute 288 sec latency 990.851 ms 2024-04-22T15:18:40.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 199050 11.56 MB/sec execute 289 sec latency 997.594 ms 2024-04-22T15:18:40.506 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:40 smithi119 ceph-mon[63156]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T15:18:40.506 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:40 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:40.506 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:40 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:40.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:40 smithi186 ceph-mon[49804]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T15:18:40.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:40 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:40.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:40 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:41.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 199315 11.54 MB/sec execute 290 sec latency 757.961 ms 2024-04-22T15:18:41.520 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:41 smithi119 ceph-mon[63156]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T15:18:41.521 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:41 smithi119 ceph-mon[63156]: pgmap v81: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 8 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s wr, 204 op/s; 101/1683 objects degraded (6.001%); 213 KiB/s, 35 objects/s recovering 2024-04-22T15:18:41.521 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:41 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:41.521 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:41 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:41.521 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:41 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:41.521 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:41 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:18:41.521 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:41 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:41.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:41 smithi186 ceph-mon[49804]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T15:18:41.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:41 smithi186 ceph-mon[49804]: pgmap v81: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 8 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s wr, 204 op/s; 101/1683 objects degraded (6.001%); 213 KiB/s, 35 objects/s recovering 2024-04-22T15:18:41.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:41 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:41.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:41 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:41.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:41 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:41.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:41 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:18:41.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:41 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:42.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 199774 11.53 MB/sec execute 291 sec latency 786.586 ms 2024-04-22T15:18:42.537 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:42.538 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:42 smithi119 ceph-mon[63156]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T15:18:42.538 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:42.538 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:42.538 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:42.538 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:42.538 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:18:42.538 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:42 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:18:42.538 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:42 smithi119 ceph-mon[63156]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-22T15:18:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:42 smithi186 ceph-mon[49804]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T15:18:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:18:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:42 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:18:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:42 smithi186 ceph-mon[49804]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-22T15:18:43.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 200070 11.53 MB/sec execute 292 sec latency 946.196 ms 2024-04-22T15:18:43.543 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:43 smithi119 ceph-mon[63156]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T15:18:43.543 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:43 smithi119 ceph-mon[63156]: pgmap v84: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 477 op/s; 101/1674 objects degraded (6.033%); 2.2 MiB/s, 35 objects/s recovering 2024-04-22T15:18:43.543 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:43 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 101/1683 objects degraded (6.001%), 5 pgs degraded (PG_DEGRADED) 2024-04-22T15:18:43.543 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:43 smithi119 ceph-mon[63156]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T15:18:43.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:43 smithi186 ceph-mon[49804]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T15:18:43.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:43 smithi186 ceph-mon[49804]: pgmap v84: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 477 op/s; 101/1674 objects degraded (6.033%); 2.2 MiB/s, 35 objects/s recovering 2024-04-22T15:18:43.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:43 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 101/1683 objects degraded (6.001%), 5 pgs degraded (PG_DEGRADED) 2024-04-22T15:18:43.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:43 smithi186 ceph-mon[49804]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T15:18:44.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 200486 11.52 MB/sec execute 293 sec latency 947.242 ms 2024-04-22T15:18:45.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 200863 11.51 MB/sec execute 294 sec latency 39.649 ms 2024-04-22T15:18:45.586 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:45 smithi119 ceph-mon[63156]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T15:18:45.586 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:45 smithi119 ceph-mon[63156]: pgmap v87: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 94 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.7 MiB/s wr, 593 op/s; 28/1695 objects degraded (1.652%); 2.0 MiB/s, 33 objects/s recovering 2024-04-22T15:18:45.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:45 smithi186 ceph-mon[49804]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T15:18:45.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:45 smithi186 ceph-mon[49804]: pgmap v87: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 94 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.7 MiB/s wr, 593 op/s; 28/1695 objects degraded (1.652%); 2.0 MiB/s, 33 objects/s recovering 2024-04-22T15:18:46.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 201167 11.50 MB/sec execute 295 sec latency 41.019 ms 2024-04-22T15:18:46.864 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:46 smithi119 ceph-mon[63156]: pgmap v88: 97 pgs: 1 peering, 96 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.9 MiB/s wr, 564 op/s; 0 B/s, 30 objects/s recovering 2024-04-22T15:18:46.865 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:46 smithi119 ceph-mon[63156]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/1695 objects degraded (1.652%), 1 pg degraded) 2024-04-22T15:18:46.865 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:46 smithi119 ceph-mon[63156]: Cluster is now healthy 2024-04-22T15:18:47.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:46 smithi186 ceph-mon[49804]: pgmap v88: 97 pgs: 1 peering, 96 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.9 MiB/s wr, 564 op/s; 0 B/s, 30 objects/s recovering 2024-04-22T15:18:47.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:46 smithi186 ceph-mon[49804]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/1695 objects degraded (1.652%), 1 pg degraded) 2024-04-22T15:18:47.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:46 smithi186 ceph-mon[49804]: Cluster is now healthy 2024-04-22T15:18:47.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 201445 11.49 MB/sec execute 296 sec latency 42.744 ms 2024-04-22T15:18:48.496 INFO:teuthology.orchestra.run.smithi119.stdout: 5 201789 11.50 MB/sec execute 297 sec latency 54.720 ms 2024-04-22T15:18:49.498 INFO:teuthology.orchestra.run.smithi119.stdout: 5 202102 11.48 MB/sec execute 298 sec latency 44.294 ms 2024-04-22T15:18:49.976 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:49 smithi119 ceph-mon[63156]: pgmap v89: 97 pgs: 1 peering, 96 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.1 MiB/s wr, 473 op/s; 0 B/s, 25 objects/s recovering 2024-04-22T15:18:50.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:49 smithi186 ceph-mon[49804]: pgmap v89: 97 pgs: 1 peering, 96 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.1 MiB/s wr, 473 op/s; 0 B/s, 25 objects/s recovering 2024-04-22T15:18:50.498 INFO:teuthology.orchestra.run.smithi119.stdout: 5 202399 11.47 MB/sec execute 299 sec latency 48.832 ms 2024-04-22T15:18:51.498 INFO:teuthology.orchestra.run.smithi119.stdout: 5 202611 11.45 MB/sec execute 300 sec latency 59.125 ms 2024-04-22T15:18:52.021 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:51 smithi119 ceph-mon[63156]: pgmap v90: 97 pgs: 97 active+clean; 339 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 6.3 MiB/s wr, 645 op/s; 0 B/s, 19 objects/s recovering 2024-04-22T15:18:52.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:51 smithi186 ceph-mon[49804]: pgmap v90: 97 pgs: 97 active+clean; 339 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 6.3 MiB/s wr, 645 op/s; 0 B/s, 19 objects/s recovering 2024-04-22T15:18:52.498 INFO:teuthology.orchestra.run.smithi119.stdout: 5 202913 11.43 MB/sec execute 301 sec latency 75.652 ms 2024-04-22T15:18:53.498 INFO:teuthology.orchestra.run.smithi119.stdout: 5 203184 11.41 MB/sec execute 302 sec latency 50.578 ms 2024-04-22T15:18:53.841 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:53 smithi119 ceph-mon[63156]: pgmap v91: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 568 op/s; 0 B/s, 17 objects/s recovering 2024-04-22T15:18:54.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:53 smithi186 ceph-mon[49804]: pgmap v91: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 568 op/s; 0 B/s, 17 objects/s recovering 2024-04-22T15:18:54.498 INFO:teuthology.orchestra.run.smithi119.stdout: 5 203753 11.43 MB/sec execute 303 sec latency 30.958 ms 2024-04-22T15:18:55.498 INFO:teuthology.orchestra.run.smithi119.stdout: 5 204301 11.44 MB/sec execute 304 sec latency 49.022 ms 2024-04-22T15:18:55.861 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:55 smithi119 ceph-mon[63156]: pgmap v92: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.5 MiB/s wr, 403 op/s; 0 B/s, 2 objects/s recovering 2024-04-22T15:18:55.862 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:55.862 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:55 smithi186 ceph-mon[49804]: pgmap v92: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.5 MiB/s wr, 403 op/s; 0 B/s, 2 objects/s recovering 2024-04-22T15:18:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:56.498 INFO:teuthology.orchestra.run.smithi119.stdout: 5 204856 11.45 MB/sec execute 305 sec latency 22.876 ms 2024-04-22T15:18:56.867 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:56 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:18:57.252 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:18:57.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:56 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:18:57.498 INFO:teuthology.orchestra.run.smithi119.stdout: 5 205401 11.48 MB/sec execute 306 sec latency 24.091 ms 2024-04-22T15:18:57.877 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:57 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:18:57.877 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:57 smithi119 ceph-mon[63156]: Upgrade: osd.1 is safe to restart 2024-04-22T15:18:57.877 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:57 smithi119 ceph-mon[63156]: pgmap v93: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 562 op/s; 0 B/s, 2 objects/s recovering 2024-04-22T15:18:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:57 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:18:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:57 smithi186 ceph-mon[49804]: Upgrade: osd.1 is safe to restart 2024-04-22T15:18:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:57 smithi186 ceph-mon[49804]: pgmap v93: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 562 op/s; 0 B/s, 2 objects/s recovering 2024-04-22T15:18:58.380 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:18:58.380 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (4m) 24s ago 13m 15.9M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:18:58.380 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (75s) 24s ago 13m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:18:58.380 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (69s) 64s ago 12m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:18:58.380 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (4m) 24s ago 13m 46.1M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:18:58.380 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (7m) 24s ago 7m 38.5M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:18:58.380 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (7m) 64s ago 7m 14.2M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (4m) 24s ago 14m 501M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (3m) 64s ago 12m 450M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (2m) 24s ago 15m 48.5M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (2m) 64s ago 12m 38.9M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (7m) 24s ago 7m 274M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (7m) 64s ago 7m 84.4M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (4m) 24s ago 13m 10.4M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (4m) 64s ago 12m 17.4M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (34s) 24s ago 12m 11.8M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (11m) 24s ago 11m 585M 1936M 17.2.0 e1d6a67b021e 44b21789ae93 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (11m) 24s ago 11m 476M 1936M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (10m) 24s ago 10m 593M 1936M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:18:58.381 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (10m) 64s ago 10m 545M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:18:58.382 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (9m) 64s ago 9m 409M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:18:58.382 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (9m) 64s ago 9m 581M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:18:58.382 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (9m) 64s ago 9m 483M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:18:58.382 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (3m) 24s ago 12m 67.6M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:18:58.498 INFO:teuthology.orchestra.run.smithi119.stdout: 5 205813 11.47 MB/sec execute 307 sec latency 55.043 ms 2024-04-22T15:18:58.882 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:58 smithi119 ceph-mon[63156]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:58.883 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:58 smithi119 ceph-mon[63156]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:59.002 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:18:59.002 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:18:59.002 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:18:59.002 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:18:59.002 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:18:59.002 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:18:59.002 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:18:59.003 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:18:59.003 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-22T15:18:59.003 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:18:59.003 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:18:59.003 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:18:59.003 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:18:59.003 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:18:59.003 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:18:59.003 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-22T15:18:59.003 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 5 2024-04-22T15:18:59.003 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:18:59.003 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:18:59.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:58 smithi186 ceph-mon[49804]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:59.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:58 smithi186 ceph-mon[49804]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:59.498 INFO:teuthology.orchestra.run.smithi119.stdout: 5 206152 11.46 MB/sec execute 308 sec latency 70.824 ms 2024-04-22T15:18:59.522 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:18:59.523 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:18:59.523 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:18:59.523 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:18:59.523 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:18:59.523 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:18:59.523 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:18:59.523 INFO:teuthology.orchestra.run.smithi119.stdout: "crash" 2024-04-22T15:18:59.523 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:18:59.523 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "7/23 daemons upgraded", 2024-04-22T15:18:59.523 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:18:59.523 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:18:59.524 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:18:59.892 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:59 smithi119 ceph-mon[63156]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:59.893 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:59 smithi119 ceph-mon[63156]: pgmap v94: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-04-22T15:18:59.893 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:59 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/4256413521' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:59.893 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:59 smithi119 ceph-mon[63156]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:59.893 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:59 smithi119 ceph-mon[63156]: Upgrade: Updating osd.1 2024-04-22T15:18:59.893 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:18:59.893 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:18:59.893 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:59.893 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:18:59 smithi119 ceph-mon[63156]: Deploying daemon osd.1 on smithi119 2024-04-22T15:19:00.188 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:19:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:59 smithi186 ceph-mon[49804]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:19:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:59 smithi186 ceph-mon[49804]: pgmap v94: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-04-22T15:19:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:59 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/4256413521' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:19:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:59 smithi186 ceph-mon[49804]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:19:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:59 smithi186 ceph-mon[49804]: Upgrade: Updating osd.1 2024-04-22T15:19:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:19:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:19:00.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:18:59 smithi186 ceph-mon[49804]: Deploying daemon osd.1 on smithi119 2024-04-22T15:19:00.499 INFO:teuthology.orchestra.run.smithi119.stdout: 5 206485 11.44 MB/sec execute 309 sec latency 52.522 ms 2024-04-22T15:19:00.910 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:00 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:00.911 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:00 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/452689236' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:19:00.911 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:00 smithi119 ceph-mon[63156]: pgmap v95: 97 pgs: 97 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 6.9 MiB/s wr, 686 op/s 2024-04-22T15:19:01.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:00 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:01.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:00 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/452689236' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:19:01.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:00 smithi186 ceph-mon[49804]: pgmap v95: 97 pgs: 97 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 6.9 MiB/s wr, 686 op/s 2024-04-22T15:19:01.499 INFO:teuthology.orchestra.run.smithi119.stdout: 5 206864 11.44 MB/sec execute 310 sec latency 41.930 ms 2024-04-22T15:19:02.500 INFO:teuthology.orchestra.run.smithi119.stdout: 5 207345 11.45 MB/sec execute 311 sec latency 43.509 ms 2024-04-22T15:19:03.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 207858 11.45 MB/sec execute 312 sec latency 28.155 ms 2024-04-22T15:19:03.932 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:03 smithi119 ceph-mon[63156]: pgmap v96: 97 pgs: 97 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.0 MiB/s wr, 510 op/s 2024-04-22T15:19:04.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:03 smithi186 ceph-mon[49804]: pgmap v96: 97 pgs: 97 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.0 MiB/s wr, 510 op/s 2024-04-22T15:19:04.500 INFO:teuthology.orchestra.run.smithi119.stdout: 5 208394 11.46 MB/sec execute 313 sec latency 39.986 ms 2024-04-22T15:19:05.500 INFO:teuthology.orchestra.run.smithi119.stdout: 5 208834 11.47 MB/sec execute 314 sec latency 41.669 ms 2024-04-22T15:19:05.703 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:05 smithi119 systemd[1]: Stopping Ceph osd.1 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:19:05.957 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:05 smithi119 ceph-mon[63156]: pgmap v97: 97 pgs: 97 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 525 op/s 2024-04-22T15:19:05.957 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:05 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[36299]: 2024-04-22T15:19:05.875+0000 7fd51da6f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:19:05.958 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:05 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[36299]: 2024-04-22T15:19:05.875+0000 7fd51da6f700 -1 osd.1 86 *** Got signal Terminated *** 2024-04-22T15:19:05.958 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:05 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[36299]: 2024-04-22T15:19:05.875+0000 7fd51da6f700 -1 osd.1 86 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:19:06.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:05 smithi186 ceph-mon[49804]: pgmap v97: 97 pgs: 97 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 525 op/s 2024-04-22T15:19:06.500 INFO:teuthology.orchestra.run.smithi119.stdout: 5 209201 11.47 MB/sec execute 315 sec latency 37.527 ms 2024-04-22T15:19:06.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:06 smithi119 ceph-mon[63156]: osd.1 marked itself down and dead 2024-04-22T15:19:07.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:06 smithi186 ceph-mon[49804]: osd.1 marked itself down and dead 2024-04-22T15:19:07.492 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:07 smithi119 podman[71411]: 2024-04-22 15:19:07.430769404 +0000 UTC m=+1.706446457 container died 44b21789ae936da0c3ef21b9a29bd7ad7e184974289adad5ca23963f56e95723 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1, GIT_BRANCH=HEAD, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, version=8, distribution-scope=public, RELEASE=HEAD, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, ceph=True, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-22T15:19:07.500 INFO:teuthology.orchestra.run.smithi119.stdout: 5 209498 11.45 MB/sec execute 316 sec latency 701.540 ms 2024-04-22T15:19:08.014 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:07 smithi119 ceph-mon[63156]: pgmap v98: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 777 op/s 2024-04-22T15:19:08.015 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:07 smithi119 ceph-mon[63156]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:19:08.015 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:07 smithi119 ceph-mon[63156]: osdmap e87: 8 total, 7 up, 8 in 2024-04-22T15:19:08.015 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:07 smithi119 podman[71411]: 2024-04-22 15:19:07.945928442 +0000 UTC m=+2.221605504 container cleanup 44b21789ae936da0c3ef21b9a29bd7ad7e184974289adad5ca23963f56e95723 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, architecture=x86_64, vendor=Red Hat, Inc., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-22T15:19:08.015 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:07 smithi119 bash[71411]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1 2024-04-22T15:19:08.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:07 smithi186 ceph-mon[49804]: pgmap v98: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 777 op/s 2024-04-22T15:19:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:07 smithi186 ceph-mon[49804]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:19:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:07 smithi186 ceph-mon[49804]: osdmap e87: 8 total, 7 up, 8 in 2024-04-22T15:19:08.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 209749 11.44 MB/sec execute 317 sec latency 1003.026 ms 2024-04-22T15:19:08.522 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:08 smithi119 podman[71426]: 2024-04-22 15:19:08.374535034 +0000 UTC m=+0.942826819 container remove 44b21789ae936da0c3ef21b9a29bd7ad7e184974289adad5ca23963f56e95723 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., io.openshift.expose-services=, name=centos-stream, architecture=x86_64, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, release=754, maintainer=Guillaume Abrioux , version=8, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-22T15:19:09.031 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:08 smithi119 podman[71513]: 2024-04-22 15:19:08.864944562 +0000 UTC m=+0.023070728 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:19:09.031 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:08 smithi119 podman[71513]: 2024-04-22 15:19:08.991965866 +0000 UTC m=+0.150092024 container create 416d8ccfc246b693b1b4d123a04a3975a18c6d197aeefb8a65c7f5e77bd956c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416) 2024-04-22T15:19:09.032 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:08 smithi119 ceph-mon[63156]: osdmap e88: 8 total, 7 up, 8 in 2024-04-22T15:19:09.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:08 smithi186 ceph-mon[49804]: osdmap e88: 8 total, 7 up, 8 in 2024-04-22T15:19:09.500 INFO:teuthology.orchestra.run.smithi119.stdout: 5 210048 11.41 MB/sec execute 318 sec latency 52.493 ms 2024-04-22T15:19:09.566 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:09 smithi119 podman[71513]: 2024-04-22 15:19:09.332629035 +0000 UTC m=+0.490755188 container init 416d8ccfc246b693b1b4d123a04a3975a18c6d197aeefb8a65c7f5e77bd956c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:19:09.566 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:09 smithi119 podman[71513]: 2024-04-22 15:19:09.342654304 +0000 UTC m=+0.500780462 container start 416d8ccfc246b693b1b4d123a04a3975a18c6d197aeefb8a65c7f5e77bd956c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0) 2024-04-22T15:19:09.566 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:09 smithi119 podman[71513]: 2024-04-22 15:19:09.563533339 +0000 UTC m=+0.721659491 container attach 416d8ccfc246b693b1b4d123a04a3975a18c6d197aeefb8a65c7f5e77bd956c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:19:09.821 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:09 smithi119 ceph-mon[63156]: pgmap v101: 97 pgs: 13 stale+active+clean, 84 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 4.8 MiB/s wr, 441 op/s 2024-04-22T15:19:09.822 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:09 smithi119 podman[71513]: 2024-04-22 15:19:09.707688082 +0000 UTC m=+0.865814240 container died 416d8ccfc246b693b1b4d123a04a3975a18c6d197aeefb8a65c7f5e77bd956c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:19:10.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:09 smithi186 ceph-mon[49804]: pgmap v101: 97 pgs: 13 stale+active+clean, 84 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 4.8 MiB/s wr, 441 op/s 2024-04-22T15:19:10.500 INFO:teuthology.orchestra.run.smithi119.stdout: 5 210308 11.41 MB/sec execute 319 sec latency 113.890 ms 2024-04-22T15:19:10.915 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:10.916 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:10.916 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:10 smithi119 podman[71539]: 2024-04-22 15:19:10.719600905 +0000 UTC m=+1.006322183 container remove 416d8ccfc246b693b1b4d123a04a3975a18c6d197aeefb8a65c7f5e77bd956c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-22T15:19:10.916 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:10 smithi119 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-22T15:19:10.916 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:10 smithi119 systemd[1]: Stopped Ceph osd.1 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:19:10.916 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:10 smithi119 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.1.service: Consumed 3min 40.825s CPU time. 2024-04-22T15:19:11.166 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:11 smithi119 systemd[1]: Starting Ceph osd.1 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:19:11.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:11.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:11.422 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:11 smithi119 podman[71640]: 2024-04-22 15:19:11.202861356 +0000 UTC m=+0.026246635 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:19:11.422 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:11 smithi119 podman[71640]: 2024-04-22 15:19:11.332396151 +0000 UTC m=+0.155781437 container create 88f8c55612032c6dde1f6b56df83777b68f620fcbd9d18c79823ebbba301e501 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate, ceph=True, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-22T15:19:11.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 210573 11.39 MB/sec execute 320 sec latency 79.137 ms 2024-04-22T15:19:11.695 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:11 smithi119 podman[71640]: 2024-04-22 15:19:11.602167831 +0000 UTC m=+0.425553091 container init 88f8c55612032c6dde1f6b56df83777b68f620fcbd9d18c79823ebbba301e501 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-22T15:19:11.695 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:11 smithi119 podman[71640]: 2024-04-22 15:19:11.612647273 +0000 UTC m=+0.436032515 container start 88f8c55612032c6dde1f6b56df83777b68f620fcbd9d18c79823ebbba301e501 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:19:11.818 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:11 smithi119 ceph-mon[63156]: pgmap v102: 97 pgs: 21 active+undersized+degraded, 3 stale+active+clean, 13 active+undersized, 60 active+clean; 334 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 7.1 MiB/s wr, 709 op/s; 143/1629 objects degraded (8.778%) 2024-04-22T15:19:11.818 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:11 smithi119 ceph-mon[63156]: Health check failed: Degraded data redundancy: 143/1629 objects degraded (8.778%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T15:19:11.818 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:11 smithi119 podman[71640]: 2024-04-22 15:19:11.71873627 +0000 UTC m=+0.542121520 container attach 88f8c55612032c6dde1f6b56df83777b68f620fcbd9d18c79823ebbba301e501 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:19:12.211 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate[71654]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:19:12.211 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate[71654]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:19:12.211 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 bash[71640]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:19:12.211 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 bash[71640]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:19:12.211 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 bash[71640]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T15:19:12.211 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 bash[71640]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T15:19:12.211 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 bash[71640]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T15:19:12.211 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 bash[71640]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T15:19:12.212 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 bash[71640]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-22T15:19:12.212 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 bash[71640]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T15:19:12.212 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 bash[71640]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-22T15:19:12.212 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate[71654]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T15:19:12.212 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate[71654]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T15:19:12.212 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate[71654]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T15:19:12.212 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate[71654]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T15:19:12.212 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate[71654]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-22T15:19:12.212 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate[71654]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T15:19:12.212 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate[71654]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-22T15:19:12.212 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:12 smithi119 podman[71640]: 2024-04-22 15:19:12.18316066 +0000 UTC m=+1.006545892 container died 88f8c55612032c6dde1f6b56df83777b68f620fcbd9d18c79823ebbba301e501 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-22T15:19:12.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:11 smithi186 ceph-mon[49804]: pgmap v102: 97 pgs: 21 active+undersized+degraded, 3 stale+active+clean, 13 active+undersized, 60 active+clean; 334 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 7.1 MiB/s wr, 709 op/s; 143/1629 objects degraded (8.778%) 2024-04-22T15:19:12.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:11 smithi186 ceph-mon[49804]: Health check failed: Degraded data redundancy: 143/1629 objects degraded (8.778%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T15:19:12.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 210924 11.38 MB/sec execute 321 sec latency 50.556 ms 2024-04-22T15:19:13.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 211150 11.37 MB/sec execute 322 sec latency 60.428 ms 2024-04-22T15:19:13.505 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:13 smithi119 podman[71781]: 2024-04-22 15:19:13.360677128 +0000 UTC m=+1.166869155 container remove 88f8c55612032c6dde1f6b56df83777b68f620fcbd9d18c79823ebbba301e501 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:19:13.756 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:13 smithi119 podman[71843]: 2024-04-22 15:19:13.492413473 +0000 UTC m=+0.020798978 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:19:13.756 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:13 smithi119 podman[71843]: 2024-04-22 15:19:13.679460577 +0000 UTC m=+0.207846079 container create 8e40a47e7598a67892f3da9da47c86367a74df70513deccf39285472d2621ec6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-22T15:19:14.011 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:13 smithi119 ceph-mon[63156]: pgmap v103: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 686 op/s; 173/1608 objects degraded (10.759%) 2024-04-22T15:19:14.266 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:14 smithi119 podman[71843]: 2024-04-22 15:19:14.050163151 +0000 UTC m=+0.578548667 container init 8e40a47e7598a67892f3da9da47c86367a74df70513deccf39285472d2621ec6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:19:14.266 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:14 smithi119 podman[71843]: 2024-04-22 15:19:14.058194652 +0000 UTC m=+0.586580154 container start 8e40a47e7598a67892f3da9da47c86367a74df70513deccf39285472d2621ec6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-22T15:19:14.266 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:14 smithi119 ceph-osd[71861]: -- 172.21.15.119:0/3933829544 <== mon.0 v2:172.21.15.119:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561e6358b680 con 0x561e63555000 2024-04-22T15:19:14.266 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:14 smithi119 bash[71843]: 8e40a47e7598a67892f3da9da47c86367a74df70513deccf39285472d2621ec6 2024-04-22T15:19:14.266 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:14 smithi119 systemd[1]: Started Ceph osd.1 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:19:14.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:13 smithi186 ceph-mon[49804]: pgmap v103: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 686 op/s; 173/1608 objects degraded (10.759%) 2024-04-22T15:19:14.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 211489 11.37 MB/sec execute 323 sec latency 49.577 ms 2024-04-22T15:19:14.778 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:14 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[71857]: 2024-04-22T15:19:14.629+0000 7f42944017c0 -1 Falling back to public interface 2024-04-22T15:19:15.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 211821 11.36 MB/sec execute 324 sec latency 57.249 ms 2024-04-22T15:19:15.550 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:15 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:15.550 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:15 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:15.550 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:15 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:19:15.550 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:15 smithi119 ceph-mon[63156]: pgmap v104: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 332 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.0 MiB/s wr, 311 op/s; 176/1578 objects degraded (11.153%) 2024-04-22T15:19:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:15 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:15 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:15.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:15 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:19:15.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:15 smithi186 ceph-mon[49804]: pgmap v104: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 332 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.0 MiB/s wr, 311 op/s; 176/1578 objects degraded (11.153%) 2024-04-22T15:19:16.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 212176 11.36 MB/sec execute 325 sec latency 43.353 ms 2024-04-22T15:19:16.815 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:16 smithi119 ceph-mon[63156]: pgmap v105: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 448 op/s; 199/1659 objects degraded (11.995%) 2024-04-22T15:19:16.816 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:16 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 199/1659 objects degraded (11.995%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T15:19:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:16 smithi186 ceph-mon[49804]: pgmap v105: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 448 op/s; 199/1659 objects degraded (11.995%) 2024-04-22T15:19:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:16 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 199/1659 objects degraded (11.995%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T15:19:17.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 212482 11.36 MB/sec execute 326 sec latency 44.846 ms 2024-04-22T15:19:18.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 212770 11.34 MB/sec execute 327 sec latency 41.595 ms 2024-04-22T15:19:19.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 213149 11.33 MB/sec execute 328 sec latency 40.137 ms 2024-04-22T15:19:19.842 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:19 smithi119 ceph-mon[63156]: pgmap v106: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.0 MiB/s wr, 407 op/s; 199/1659 objects degraded (11.995%) 2024-04-22T15:19:20.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:19 smithi186 ceph-mon[49804]: pgmap v106: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.0 MiB/s wr, 407 op/s; 199/1659 objects degraded (11.995%) 2024-04-22T15:19:20.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 213414 11.31 MB/sec execute 329 sec latency 56.279 ms 2024-04-22T15:19:21.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 213757 11.31 MB/sec execute 330 sec latency 52.166 ms 2024-04-22T15:19:21.620 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:21 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[71857]: 2024-04-22T15:19:21.580+0000 7f42944017c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-22T15:19:21.879 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:21 smithi119 ceph-mon[63156]: pgmap v107: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.6 MiB/s wr, 571 op/s; 191/1710 objects degraded (11.170%) 2024-04-22T15:19:22.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:21 smithi186 ceph-mon[49804]: pgmap v107: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.6 MiB/s wr, 571 op/s; 191/1710 objects degraded (11.170%) 2024-04-22T15:19:22.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 214134 11.30 MB/sec execute 331 sec latency 45.122 ms 2024-04-22T15:19:22.907 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:22 smithi119 ceph-mon[63156]: pgmap v108: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.9 MiB/s wr, 388 op/s; 193/1716 objects degraded (11.247%) 2024-04-22T15:19:22.907 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:22 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 193/1716 objects degraded (11.247%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T15:19:22.908 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:22 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[71857]: 2024-04-22T15:19:22.840+0000 7f42944017c0 -1 osd.1 86 log_to_monitors true 2024-04-22T15:19:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:22 smithi186 ceph-mon[49804]: pgmap v108: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.9 MiB/s wr, 388 op/s; 193/1716 objects degraded (11.247%) 2024-04-22T15:19:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:22 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 193/1716 objects degraded (11.247%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T15:19:23.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 214588 11.30 MB/sec execute 332 sec latency 33.276 ms 2024-04-22T15:19:23.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:23 smithi119 ceph-mon[63156]: from='osd.1 [v2:172.21.15.119:6810/1547006003,v1:172.21.15.119:6811/1547006003]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:19:23.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:23.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:24.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:23 smithi186 ceph-mon[49804]: from='osd.1 [v2:172.21.15.119:6810/1547006003,v1:172.21.15.119:6811/1547006003]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:19:24.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:24.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:24.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 214762 11.28 MB/sec execute 333 sec latency 176.787 ms 2024-04-22T15:19:24.541 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:19:24 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[71857]: 2024-04-22T15:19:24.376+0000 7f4281851700 -1 osd.1 86 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:19:25.044 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:24 smithi119 ceph-mon[63156]: from='osd.1 [v2:172.21.15.119:6810/1547006003,v1:172.21.15.119:6811/1547006003]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T15:19:25.044 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:24 smithi119 ceph-mon[63156]: osdmap e89: 8 total, 7 up, 8 in 2024-04-22T15:19:25.045 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:24 smithi119 ceph-mon[63156]: from='osd.1 [v2:172.21.15.119:6810/1547006003,v1:172.21.15.119:6811/1547006003]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:19:25.045 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:24 smithi119 ceph-mon[63156]: from='osd.1 [v2:172.21.15.119:6810/1547006003,v1:172.21.15.119:6811/1547006003]' entity='osd.1' 2024-04-22T15:19:25.045 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:24 smithi119 ceph-mon[63156]: pgmap v110: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 459 op/s; 194/1695 objects degraded (11.445%) 2024-04-22T15:19:25.045 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:25.045 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:25.045 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:25.045 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:24 smithi186 ceph-mon[49804]: from='osd.1 [v2:172.21.15.119:6810/1547006003,v1:172.21.15.119:6811/1547006003]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T15:19:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:24 smithi186 ceph-mon[49804]: osdmap e89: 8 total, 7 up, 8 in 2024-04-22T15:19:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:24 smithi186 ceph-mon[49804]: from='osd.1 [v2:172.21.15.119:6810/1547006003,v1:172.21.15.119:6811/1547006003]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:19:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:24 smithi186 ceph-mon[49804]: from='osd.1 [v2:172.21.15.119:6810/1547006003,v1:172.21.15.119:6811/1547006003]' entity='osd.1' 2024-04-22T15:19:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:24 smithi186 ceph-mon[49804]: pgmap v110: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 459 op/s; 194/1695 objects degraded (11.445%) 2024-04-22T15:19:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:24 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:24 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:24 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:24 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:25.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 215062 11.29 MB/sec execute 334 sec latency 108.437 ms 2024-04-22T15:19:26.137 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:25 smithi119 ceph-mon[63156]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:19:26.137 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:25 smithi119 ceph-mon[63156]: osd.1 [v2:172.21.15.119:6810/1547006003,v1:172.21.15.119:6811/1547006003] boot 2024-04-22T15:19:26.137 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:25 smithi119 ceph-mon[63156]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T15:19:26.137 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:19:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:25 smithi186 ceph-mon[49804]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:19:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:25 smithi186 ceph-mon[49804]: osd.1 [v2:172.21.15.119:6810/1547006003,v1:172.21.15.119:6811/1547006003] boot 2024-04-22T15:19:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:25 smithi186 ceph-mon[49804]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T15:19:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:19:26.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 215105 11.26 MB/sec execute 335 sec latency 1108.467 ms 2024-04-22T15:19:27.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 215137 11.22 MB/sec execute 336 sec latency 2108.494 ms 2024-04-22T15:19:27.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:27 smithi119 ceph-mon[63156]: osdmap e91: 8 total, 8 up, 8 in 2024-04-22T15:19:27.694 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:27 smithi119 ceph-mon[63156]: pgmap v113: 97 pgs: 23 peering, 10 active+undersized+degraded, 6 active+undersized, 58 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 3.9 MiB/s wr, 381 op/s; 78/1629 objects degraded (4.788%) 2024-04-22T15:19:27.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:27 smithi186 ceph-mon[49804]: osdmap e91: 8 total, 8 up, 8 in 2024-04-22T15:19:27.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:27 smithi186 ceph-mon[49804]: pgmap v113: 97 pgs: 23 peering, 10 active+undersized+degraded, 6 active+undersized, 58 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 3.9 MiB/s wr, 381 op/s; 78/1629 objects degraded (4.788%) 2024-04-22T15:19:28.455 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:28 smithi119 ceph-mon[63156]: osdmap e92: 8 total, 8 up, 8 in 2024-04-22T15:19:28.501 INFO:teuthology.orchestra.run.smithi119.stdout: 5 215465 11.21 MB/sec execute 337 sec latency 2331.837 ms 2024-04-22T15:19:28.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:28 smithi186 ceph-mon[49804]: osdmap e92: 8 total, 8 up, 8 in 2024-04-22T15:19:29.467 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:29 smithi119 ceph-mon[63156]: osdmap e93: 8 total, 8 up, 8 in 2024-04-22T15:19:29.468 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:29 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 78/1629 objects degraded (4.788%), 10 pgs degraded (PG_DEGRADED) 2024-04-22T15:19:29.468 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:29 smithi119 ceph-mon[63156]: pgmap v116: 97 pgs: 23 peering, 10 active+undersized+degraded, 6 active+undersized, 58 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.9 MiB/s wr, 514 op/s; 78/1629 objects degraded (4.788%) 2024-04-22T15:19:29.468 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:29 smithi119 ceph-mon[63156]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T15:19:29.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 215778 11.20 MB/sec execute 338 sec latency 795.823 ms 2024-04-22T15:19:29.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:29 smithi186 ceph-mon[49804]: osdmap e93: 8 total, 8 up, 8 in 2024-04-22T15:19:29.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:29 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 78/1629 objects degraded (4.788%), 10 pgs degraded (PG_DEGRADED) 2024-04-22T15:19:29.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:29 smithi186 ceph-mon[49804]: pgmap v116: 97 pgs: 23 peering, 10 active+undersized+degraded, 6 active+undersized, 58 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.9 MiB/s wr, 514 op/s; 78/1629 objects degraded (4.788%) 2024-04-22T15:19:29.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:29 smithi186 ceph-mon[49804]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T15:19:30.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 216164 11.21 MB/sec execute 339 sec latency 799.453 ms 2024-04-22T15:19:30.743 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:19:31.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:31.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:31.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:19:31.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:19:31.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:31.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:31.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:19:31.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:19:31.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:19:31.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:19:31.262 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:19:31.262 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:19:31.262 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-22T15:19:31.262 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T15:19:31.262 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: pgmap v119: 97 pgs: 1 active+undersized+remapped, 6 peering, 1 active+undersized+degraded, 1 active+recovery_wait+degraded, 88 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s wr, 422 op/s; 32/1752 objects degraded (1.826%); 1.1 MiB/s, 65 keys/s, 50 objects/s recovering 2024-04-22T15:19:31.262 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:31 smithi119 ceph-mon[63156]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:19:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:19:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:19:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:19:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:19:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:19:31.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:19:31.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:19:31.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:19:31.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-22T15:19:31.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T15:19:31.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: pgmap v119: 97 pgs: 1 active+undersized+remapped, 6 peering, 1 active+undersized+degraded, 1 active+recovery_wait+degraded, 88 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s wr, 422 op/s; 32/1752 objects degraded (1.826%); 1.1 MiB/s, 65 keys/s, 50 objects/s recovering 2024-04-22T15:19:31.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:31 smithi186 ceph-mon[49804]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:19:31.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 216573 11.20 MB/sec execute 340 sec latency 750.319 ms 2024-04-22T15:19:31.799 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:19:31.799 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (5m) 8s ago 14m 14.4M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:19:31.799 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (108s) 8s ago 14m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:19:31.799 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (102s) 97s ago 13m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:19:31.799 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (4m) 8s ago 13m 47.6M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:19:31.799 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (8m) 8s ago 8m 38.5M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:19:31.799 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (8m) 97s ago 8m 14.2M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:19:31.799 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (4m) 8s ago 15m 506M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (3m) 97s ago 13m 450M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (3m) 8s ago 15m 50.3M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (2m) 97s ago 13m 38.9M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (7m) 8s ago 7m 280M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (7m) 97s ago 7m 84.4M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (4m) 8s ago 13m 10.4M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (4m) 97s ago 13m 17.4M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (68s) 8s ago 12m 249M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (18s) 8s ago 12m 12.8M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (11m) 8s ago 11m 519M 1936M 17.2.0 e1d6a67b021e 216994a26569 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (11m) 8s ago 11m 376M 1936M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (10m) 97s ago 10m 545M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (10m) 97s ago 10m 409M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (10m) 97s ago 10m 581M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:19:31.800 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (9m) 97s ago 9m 483M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:19:31.801 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (3m) 8s ago 12m 67.7M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:19:32.449 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:19:32.449 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:19:32.449 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:19:32.449 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:19:32.449 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:19:32.449 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:19:32.449 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:19:32.449 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:19:32.450 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-22T15:19:32.450 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:19:32.450 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:19:32.450 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:19:32.450 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:19:32.450 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:19:32.450 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:19:32.450 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-22T15:19:32.450 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 6 2024-04-22T15:19:32.450 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:19:32.450 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:19:32.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 216962 11.20 MB/sec execute 341 sec latency 77.311 ms 2024-04-22T15:19:32.532 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:32 smithi119 ceph-mon[63156]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:19:32.532 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:32 smithi119 ceph-mon[63156]: osdmap e96: 8 total, 8 up, 8 in 2024-04-22T15:19:32.532 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:32 smithi119 ceph-mon[63156]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:19:32.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:32 smithi186 ceph-mon[49804]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:19:32.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:32 smithi186 ceph-mon[49804]: osdmap e96: 8 total, 8 up, 8 in 2024-04-22T15:19:32.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:32 smithi186 ceph-mon[49804]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:19:32.992 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:19:32.992 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:19:32.992 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:19:32.992 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:19:32.992 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:19:32.992 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:19:32.992 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:19:32.992 INFO:teuthology.orchestra.run.smithi119.stdout: "crash" 2024-04-22T15:19:32.993 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:19:32.993 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "8/23 daemons upgraded", 2024-04-22T15:19:32.993 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:19:32.993 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:19:32.993 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:19:33.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 217374 11.21 MB/sec execute 342 sec latency 1010.044 ms 2024-04-22T15:19:33.543 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:33 smithi119 ceph-mon[63156]: osdmap e97: 8 total, 8 up, 8 in 2024-04-22T15:19:33.544 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:33 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/3149687348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:19:33.544 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:33 smithi119 ceph-mon[63156]: pgmap v122: 97 pgs: 1 active+undersized+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 90 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.6 MiB/s wr, 436 op/s; 22/1740 objects degraded (1.264%); 1.1 MiB/s, 65 keys/s, 51 objects/s recovering 2024-04-22T15:19:33.661 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_WARN Degraded data redundancy: 22/1740 objects degraded (1.264%), 2 pgs degraded 2024-04-22T15:19:33.662 INFO:teuthology.orchestra.run.smithi119.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 22/1740 objects degraded (1.264%), 2 pgs degraded 2024-04-22T15:19:33.662 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.0 is active+recovering+undersized+degraded+remapped, acting [2,6] 2024-04-22T15:19:33.662 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1c is active+recovery_wait+degraded, acting [5,4,1] 2024-04-22T15:19:33.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:33 smithi186 ceph-mon[49804]: osdmap e97: 8 total, 8 up, 8 in 2024-04-22T15:19:33.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:33 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/3149687348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:19:33.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:33 smithi186 ceph-mon[49804]: pgmap v122: 97 pgs: 1 active+undersized+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 90 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.6 MiB/s wr, 436 op/s; 22/1740 objects degraded (1.264%); 1.1 MiB/s, 65 keys/s, 51 objects/s recovering 2024-04-22T15:19:34.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 217823 11.21 MB/sec execute 343 sec latency 36.059 ms 2024-04-22T15:19:34.554 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:34 smithi119 ceph-mon[63156]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:19:34.554 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:34 smithi119 ceph-mon[63156]: osdmap e98: 8 total, 8 up, 8 in 2024-04-22T15:19:34.554 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:34 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 22/1740 objects degraded (1.264%), 2 pgs degraded (PG_DEGRADED) 2024-04-22T15:19:34.555 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:34 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/3280066131' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:19:34.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:34 smithi186 ceph-mon[49804]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:19:34.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:34 smithi186 ceph-mon[49804]: osdmap e98: 8 total, 8 up, 8 in 2024-04-22T15:19:34.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:34 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 22/1740 objects degraded (1.264%), 2 pgs degraded (PG_DEGRADED) 2024-04-22T15:19:34.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:34 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/3280066131' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:19:35.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 218228 11.20 MB/sec execute 344 sec latency 58.241 ms 2024-04-22T15:19:35.564 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:35 smithi119 ceph-mon[63156]: pgmap v124: 97 pgs: 1 active+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 94 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 5.6 MiB/s wr, 590 op/s; 16/1824 objects degraded (0.877%); 1.8 MiB/s, 38 objects/s recovering 2024-04-22T15:19:35.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:35 smithi186 ceph-mon[49804]: pgmap v124: 97 pgs: 1 active+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 94 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 5.6 MiB/s wr, 590 op/s; 16/1824 objects degraded (0.877%); 1.8 MiB/s, 38 objects/s recovering 2024-04-22T15:19:36.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 218653 11.20 MB/sec execute 345 sec latency 32.596 ms 2024-04-22T15:19:36.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:36 smithi119 ceph-mon[63156]: pgmap v125: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.8 MiB/s wr, 446 op/s; 683 KiB/s, 8 objects/s recovering 2024-04-22T15:19:36.834 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:36 smithi119 ceph-mon[63156]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/1824 objects degraded (0.877%), 1 pg degraded) 2024-04-22T15:19:36.834 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:36 smithi119 ceph-mon[63156]: Cluster is now healthy 2024-04-22T15:19:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:36 smithi186 ceph-mon[49804]: pgmap v125: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.8 MiB/s wr, 446 op/s; 683 KiB/s, 8 objects/s recovering 2024-04-22T15:19:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:36 smithi186 ceph-mon[49804]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/1824 objects degraded (0.877%), 1 pg degraded) 2024-04-22T15:19:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:36 smithi186 ceph-mon[49804]: Cluster is now healthy 2024-04-22T15:19:37.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 219036 11.20 MB/sec execute 346 sec latency 41.754 ms 2024-04-22T15:19:38.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 219488 11.20 MB/sec execute 347 sec latency 43.765 ms 2024-04-22T15:19:39.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 219858 11.20 MB/sec execute 348 sec latency 49.766 ms 2024-04-22T15:19:39.883 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:39 smithi119 ceph-mon[63156]: pgmap v126: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.8 MiB/s wr, 357 op/s; 559 KiB/s, 6 objects/s recovering 2024-04-22T15:19:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:39 smithi186 ceph-mon[49804]: pgmap v126: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.8 MiB/s wr, 357 op/s; 559 KiB/s, 6 objects/s recovering 2024-04-22T15:19:40.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 220309 11.19 MB/sec execute 349 sec latency 49.039 ms 2024-04-22T15:19:40.887 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:40 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:40.888 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:40 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:41.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:40 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:41.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:40 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:41.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 220976 11.23 MB/sec execute 350 sec latency 27.690 ms 2024-04-22T15:19:41.902 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:41 smithi119 ceph-mon[63156]: pgmap v127: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 7.2 MiB/s wr, 734 op/s; 492 KiB/s, 5 objects/s recovering 2024-04-22T15:19:42.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:41 smithi186 ceph-mon[49804]: pgmap v127: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 7.2 MiB/s wr, 734 op/s; 492 KiB/s, 5 objects/s recovering 2024-04-22T15:19:42.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 221511 11.23 MB/sec execute 351 sec latency 40.009 ms 2024-04-22T15:19:43.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 221831 11.22 MB/sec execute 352 sec latency 40.061 ms 2024-04-22T15:19:43.834 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:43 smithi119 ceph-mon[63156]: pgmap v128: 97 pgs: 97 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.0 MiB/s wr, 613 op/s; 410 KiB/s, 4 objects/s recovering 2024-04-22T15:19:44.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:43 smithi186 ceph-mon[49804]: pgmap v128: 97 pgs: 97 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.0 MiB/s wr, 613 op/s; 410 KiB/s, 4 objects/s recovering 2024-04-22T15:19:44.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 222194 11.22 MB/sec execute 353 sec latency 33.992 ms 2024-04-22T15:19:45.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 222547 11.21 MB/sec execute 354 sec latency 38.155 ms 2024-04-22T15:19:45.976 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:45 smithi119 ceph-mon[63156]: pgmap v129: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 570 op/s; 362 KiB/s, 4 objects/s recovering 2024-04-22T15:19:45.976 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:45 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:45.977 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:45 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:19:46.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:45 smithi186 ceph-mon[49804]: pgmap v129: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 570 op/s; 362 KiB/s, 4 objects/s recovering 2024-04-22T15:19:46.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:45 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:46.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:45 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:19:46.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 222909 11.21 MB/sec execute 355 sec latency 53.817 ms 2024-04-22T15:19:47.015 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:46 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:19:47.015 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:46 smithi119 ceph-mon[63156]: Upgrade: osd.2 is safe to restart 2024-04-22T15:19:47.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:46 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:19:47.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:46 smithi186 ceph-mon[49804]: Upgrade: osd.2 is safe to restart 2024-04-22T15:19:47.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 223302 11.21 MB/sec execute 356 sec latency 62.397 ms 2024-04-22T15:19:48.030 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:47 smithi119 ceph-mon[63156]: pgmap v130: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 702 op/s; 0 B/s, 1 objects/s recovering 2024-04-22T15:19:48.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:47 smithi186 ceph-mon[49804]: pgmap v130: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 702 op/s; 0 B/s, 1 objects/s recovering 2024-04-22T15:19:48.502 INFO:teuthology.orchestra.run.smithi119.stdout: 5 223776 11.20 MB/sec execute 357 sec latency 47.254 ms 2024-04-22T15:19:49.503 INFO:teuthology.orchestra.run.smithi119.stdout: 5 224095 11.20 MB/sec execute 358 sec latency 53.604 ms 2024-04-22T15:19:49.565 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:49 smithi119 ceph-mon[63156]: Upgrade: Updating osd.2 2024-04-22T15:19:49.565 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:49 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:49.566 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:49 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:19:49.566 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:49 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:19:49.566 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:49 smithi119 ceph-mon[63156]: Deploying daemon osd.2 on smithi119 2024-04-22T15:19:49.566 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:49 smithi119 ceph-mon[63156]: pgmap v131: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 566 op/s 2024-04-22T15:19:49.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:49 smithi186 ceph-mon[49804]: Upgrade: Updating osd.2 2024-04-22T15:19:49.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:49 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:19:49.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:49 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:19:49.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:49 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:19:49.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:49 smithi186 ceph-mon[49804]: Deploying daemon osd.2 on smithi119 2024-04-22T15:19:49.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:49 smithi186 ceph-mon[49804]: pgmap v131: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 566 op/s 2024-04-22T15:19:50.509 INFO:teuthology.orchestra.run.smithi119.stdout: 5 224497 11.21 MB/sec execute 359 sec latency 55.254 ms 2024-04-22T15:19:51.509 INFO:teuthology.orchestra.run.smithi119.stdout: 5 224798 11.20 MB/sec execute 360 sec latency 56.995 ms 2024-04-22T15:19:51.886 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:51 smithi119 ceph-mon[63156]: pgmap v132: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.5 MiB/s wr, 782 op/s 2024-04-22T15:19:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:51 smithi186 ceph-mon[49804]: pgmap v132: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.5 MiB/s wr, 782 op/s 2024-04-22T15:19:52.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 225098 11.20 MB/sec execute 361 sec latency 59.254 ms 2024-04-22T15:19:53.509 INFO:teuthology.orchestra.run.smithi119.stdout: 5 225283 11.17 MB/sec execute 362 sec latency 84.928 ms 2024-04-22T15:19:53.957 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:53 smithi119 ceph-mon[63156]: pgmap v133: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 509 op/s 2024-04-22T15:19:54.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:53 smithi186 ceph-mon[49804]: pgmap v133: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 509 op/s 2024-04-22T15:19:54.462 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:54 smithi119 systemd[1]: Stopping Ceph osd.2 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:19:54.509 INFO:teuthology.orchestra.run.smithi119.stdout: 5 225545 11.16 MB/sec execute 363 sec latency 47.724 ms 2024-04-22T15:19:54.712 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:54 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2[40001]: 2024-04-22T15:19:54.586+0000 7f8dc8570700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:19:54.713 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:54 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2[40001]: 2024-04-22T15:19:54.586+0000 7f8dc8570700 -1 osd.2 98 *** Got signal Terminated *** 2024-04-22T15:19:54.713 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:54 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2[40001]: 2024-04-22T15:19:54.586+0000 7f8dc8570700 -1 osd.2 98 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:19:54.965 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:54 smithi119 ceph-mon[63156]: osd.2 marked itself down and dead 2024-04-22T15:19:55.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:54 smithi186 ceph-mon[49804]: osd.2 marked itself down and dead 2024-04-22T15:19:55.467 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:55 smithi119 podman[74002]: 2024-04-22 15:19:55.451551599 +0000 UTC m=+1.040386899 container died 216994a265690159b84519be89046f0c6f45e5187633c6771fe79f500e2a8b91 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, RELEASE=HEAD, io.buildah.version=1.19.8, name=centos-stream, distribution-scope=public, com.redhat.component=centos-stream-container) 2024-04-22T15:19:55.509 INFO:teuthology.orchestra.run.smithi119.stdout: 5 225861 11.14 MB/sec execute 364 sec latency 102.544 ms 2024-04-22T15:19:55.976 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:55 smithi119 ceph-mon[63156]: pgmap v134: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 527 op/s 2024-04-22T15:19:55.976 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:55 smithi119 ceph-mon[63156]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:19:55.976 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:55 smithi119 ceph-mon[63156]: osdmap e99: 8 total, 7 up, 8 in 2024-04-22T15:19:55.977 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:55 smithi186 ceph-mon[49804]: pgmap v134: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 527 op/s 2024-04-22T15:19:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:55 smithi186 ceph-mon[49804]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:19:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:55 smithi186 ceph-mon[49804]: osdmap e99: 8 total, 7 up, 8 in 2024-04-22T15:19:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:56.478 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:56 smithi119 podman[74002]: 2024-04-22 15:19:56.383527215 +0000 UTC m=+1.972362516 container cleanup 216994a265690159b84519be89046f0c6f45e5187633c6771fe79f500e2a8b91 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, version=8, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, release=754, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, vendor=Red Hat, Inc., name=centos-stream, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.component=centos-stream-container) 2024-04-22T15:19:56.478 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:56 smithi119 bash[74002]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2 2024-04-22T15:19:56.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 226270 11.15 MB/sec execute 365 sec latency 400.274 ms 2024-04-22T15:19:56.741 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:56 smithi119 podman[74020]: 2024-04-22 15:19:56.721415417 +0000 UTC m=+1.268548128 container remove 216994a265690159b84519be89046f0c6f45e5187633c6771fe79f500e2a8b91 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, architecture=x86_64, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, version=8, distribution-scope=public, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, vcs-type=git) 2024-04-22T15:19:57.022 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:56 smithi119 ceph-mon[63156]: osdmap e100: 8 total, 7 up, 8 in 2024-04-22T15:19:57.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:56 smithi186 ceph-mon[49804]: osdmap e100: 8 total, 7 up, 8 in 2024-04-22T15:19:57.509 INFO:teuthology.orchestra.run.smithi119.stdout: 5 226644 11.15 MB/sec execute 366 sec latency 51.892 ms 2024-04-22T15:19:57.531 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:57 smithi119 podman[74107]: 2024-04-22 15:19:57.21474245 +0000 UTC m=+0.029673323 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:19:57.531 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:57 smithi119 podman[74107]: 2024-04-22 15:19:57.368830825 +0000 UTC m=+0.183761679 container create 0da052b9a637159a08efac62a650cee9d8a677304327b1dc45a96be23c2dbab1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-22T15:19:57.782 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:57 smithi119 podman[74107]: 2024-04-22 15:19:57.739854904 +0000 UTC m=+0.554785772 container init 0da052b9a637159a08efac62a650cee9d8a677304327b1dc45a96be23c2dbab1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-22T15:19:57.782 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:57 smithi119 podman[74107]: 2024-04-22 15:19:57.7467027 +0000 UTC m=+0.561633553 container start 0da052b9a637159a08efac62a650cee9d8a677304327b1dc45a96be23c2dbab1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True) 2024-04-22T15:19:58.033 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:57 smithi119 ceph-mon[63156]: pgmap v137: 97 pgs: 18 peering, 2 stale+active+clean, 77 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 568 op/s 2024-04-22T15:19:58.033 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:57 smithi119 ceph-mon[63156]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-22T15:19:58.034 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:57 smithi119 podman[74107]: 2024-04-22 15:19:57.907546857 +0000 UTC m=+0.722477721 container attach 0da052b9a637159a08efac62a650cee9d8a677304327b1dc45a96be23c2dbab1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-22T15:19:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:57 smithi186 ceph-mon[49804]: pgmap v137: 97 pgs: 18 peering, 2 stale+active+clean, 77 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 568 op/s 2024-04-22T15:19:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:57 smithi186 ceph-mon[49804]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-22T15:19:58.284 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:58 smithi119 podman[74107]: 2024-04-22 15:19:58.196832869 +0000 UTC m=+1.011763724 container died 0da052b9a637159a08efac62a650cee9d8a677304327b1dc45a96be23c2dbab1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:19:58.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 227055 11.14 MB/sec execute 367 sec latency 37.981 ms 2024-04-22T15:19:59.305 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:59 smithi119 podman[74165]: 2024-04-22 15:19:59.140509547 +0000 UTC m=+0.930708940 container remove 0da052b9a637159a08efac62a650cee9d8a677304327b1dc45a96be23c2dbab1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-04-22T15:19:59.305 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:59 smithi119 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-22T15:19:59.305 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:59 smithi119 systemd[1]: Stopped Ceph osd.2 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:19:59.305 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:59 smithi119 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.2.service: Consumed 2min 52.902s CPU time. 2024-04-22T15:19:59.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 227574 11.14 MB/sec execute 368 sec latency 54.410 ms 2024-04-22T15:19:59.556 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:19:59 smithi119 ceph-mon[63156]: pgmap v138: 97 pgs: 18 peering, 2 stale+active+clean, 77 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 2.9 MiB/s wr, 244 op/s 2024-04-22T15:19:59.557 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:59 smithi119 systemd[1]: Starting Ceph osd.2 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:19:59.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:19:59 smithi186 ceph-mon[49804]: pgmap v138: 97 pgs: 18 peering, 2 stale+active+clean, 77 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 2.9 MiB/s wr, 244 op/s 2024-04-22T15:19:59.809 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:59 smithi119 podman[74262]: 2024-04-22 15:19:59.67313305 +0000 UTC m=+0.022899995 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:20:00.061 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:19:59 smithi119 podman[74262]: 2024-04-22 15:19:59.826026684 +0000 UTC m=+0.175793610 container create ce76029881243961b30402e6126faf4f8d57787d462c069783f66f4b7f003cc7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2) 2024-04-22T15:20:00.311 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 podman[74262]: 2024-04-22 15:20:00.21460474 +0000 UTC m=+0.564371670 container init ce76029881243961b30402e6126faf4f8d57787d462c069783f66f4b7f003cc7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416) 2024-04-22T15:20:00.311 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 podman[74262]: 2024-04-22 15:20:00.220129713 +0000 UTC m=+0.569896626 container start ce76029881243961b30402e6126faf4f8d57787d462c069783f66f4b7f003cc7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240416) 2024-04-22T15:20:00.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 228006 11.16 MB/sec execute 369 sec latency 36.868 ms 2024-04-22T15:20:00.562 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 podman[74262]: 2024-04-22 15:20:00.390422735 +0000 UTC m=+0.740189657 container attach ce76029881243961b30402e6126faf4f8d57787d462c069783f66f4b7f003cc7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-22T15:20:00.563 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-mon[63156]: Health detail: HEALTH_WARN 1 osds down; Reduced data availability: 1 pg inactive, 2 pgs peering 2024-04-22T15:20:00.563 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-mon[63156]: [WRN] OSD_DOWN: 1 osds down 2024-04-22T15:20:00.563 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-mon[63156]: osd.2 (root=default,host=smithi119) is down 2024-04-22T15:20:00.563 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-mon[63156]: [WRN] PG_AVAILABILITY: Reduced data availability: 1 pg inactive, 2 pgs peering 2024-04-22T15:20:00.563 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-mon[63156]: pg 2.e is stuck peering for 2m, current state peering, last acting [3,7] 2024-04-22T15:20:00.563 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-mon[63156]: pg 4.2 is stuck peering for 8m, current state peering, last acting [5,4] 2024-04-22T15:20:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:00 smithi186 ceph-mon[49804]: Health detail: HEALTH_WARN 1 osds down; Reduced data availability: 1 pg inactive, 2 pgs peering 2024-04-22T15:20:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:00 smithi186 ceph-mon[49804]: [WRN] OSD_DOWN: 1 osds down 2024-04-22T15:20:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:00 smithi186 ceph-mon[49804]: osd.2 (root=default,host=smithi119) is down 2024-04-22T15:20:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:00 smithi186 ceph-mon[49804]: [WRN] PG_AVAILABILITY: Reduced data availability: 1 pg inactive, 2 pgs peering 2024-04-22T15:20:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:00 smithi186 ceph-mon[49804]: pg 2.e is stuck peering for 2m, current state peering, last acting [3,7] 2024-04-22T15:20:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:00 smithi186 ceph-mon[49804]: pg 4.2 is stuck peering for 8m, current state peering, last acting [5,4] 2024-04-22T15:20:01.078 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate[74276]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:20:01.078 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate[74276]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:20:01.078 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate[74276]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T15:20:01.078 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate[74276]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T15:20:01.078 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate[74276]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T15:20:01.078 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate[74276]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T15:20:01.078 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate[74276]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-22T15:20:01.078 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate[74276]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T15:20:01.079 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate[74276]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-22T15:20:01.079 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 bash[74262]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:20:01.079 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 bash[74262]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:20:01.079 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 bash[74262]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T15:20:01.079 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 bash[74262]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T15:20:01.079 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 bash[74262]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T15:20:01.079 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 bash[74262]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T15:20:01.079 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 bash[74262]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-22T15:20:01.079 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 bash[74262]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T15:20:01.079 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 bash[74262]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-22T15:20:01.079 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:00 smithi119 podman[74262]: 2024-04-22 15:20:00.971430479 +0000 UTC m=+1.321197398 container died ce76029881243961b30402e6126faf4f8d57787d462c069783f66f4b7f003cc7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:20:01.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 228438 11.16 MB/sec execute 370 sec latency 46.300 ms 2024-04-22T15:20:01.580 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:01 smithi119 ceph-mon[63156]: pgmap v139: 97 pgs: 15 active+undersized+degraded, 4 peering, 7 active+undersized, 71 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.9 KiB/s rd, 5.8 MiB/s wr, 603 op/s; 127/1719 objects degraded (7.388%) 2024-04-22T15:20:01.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:01 smithi186 ceph-mon[49804]: pgmap v139: 97 pgs: 15 active+undersized+degraded, 4 peering, 7 active+undersized, 71 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.9 KiB/s rd, 5.8 MiB/s wr, 603 op/s; 127/1719 objects degraded (7.388%) 2024-04-22T15:20:02.087 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:02 smithi119 podman[74404]: 2024-04-22 15:20:02.026891765 +0000 UTC m=+1.048171476 container remove ce76029881243961b30402e6126faf4f8d57787d462c069783f66f4b7f003cc7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-activate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True) 2024-04-22T15:20:02.338 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:02 smithi119 podman[74461]: 2024-04-22 15:20:02.203239853 +0000 UTC m=+0.029309460 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:20:02.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 228848 11.16 MB/sec execute 371 sec latency 36.999 ms 2024-04-22T15:20:02.589 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:02 smithi119 podman[74461]: 2024-04-22 15:20:02.341568538 +0000 UTC m=+0.167638127 container create 14287157a66ebaa3b222a19a6bdb8c18aaead191ccd5fc653ac789fee0bbb343 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD) 2024-04-22T15:20:02.589 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:02 smithi119 ceph-mon[63156]: Health check failed: Degraded data redundancy: 127/1719 objects degraded (7.388%), 15 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:02.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:02 smithi186 ceph-mon[49804]: Health check failed: Degraded data redundancy: 127/1719 objects degraded (7.388%), 15 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:02.846 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:02 smithi119 podman[74461]: 2024-04-22 15:20:02.738576039 +0000 UTC m=+0.564645657 container init 14287157a66ebaa3b222a19a6bdb8c18aaead191ccd5fc653ac789fee0bbb343 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD) 2024-04-22T15:20:02.847 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:02 smithi119 podman[74461]: 2024-04-22 15:20:02.751375342 +0000 UTC m=+0.577444936 container start 14287157a66ebaa3b222a19a6bdb8c18aaead191ccd5fc653ac789fee0bbb343 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-22T15:20:02.847 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:02 smithi119 ceph-osd[74479]: -- 172.21.15.119:0/228649985 <== mon.0 v2:172.21.15.119:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558c0e787680 con 0x558c0e751000 2024-04-22T15:20:03.101 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:02 smithi119 bash[74461]: 14287157a66ebaa3b222a19a6bdb8c18aaead191ccd5fc653ac789fee0bbb343 2024-04-22T15:20:03.101 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:02 smithi119 systemd[1]: Started Ceph osd.2 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:20:03.352 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:03 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2[74475]: 2024-04-22T15:20:03.332+0000 7fd428d757c0 -1 Falling back to public interface 2024-04-22T15:20:03.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 229106 11.14 MB/sec execute 372 sec latency 50.118 ms 2024-04-22T15:20:03.607 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:03 smithi119 ceph-mon[63156]: pgmap v140: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.0 MiB/s wr, 610 op/s; 150/1764 objects degraded (8.503%) 2024-04-22T15:20:03.607 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:03 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:03.607 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:03 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:03.608 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:03 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:20:03.608 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:03 smithi119 ceph-mon[63156]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T15:20:03.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:03 smithi186 ceph-mon[49804]: pgmap v140: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.0 MiB/s wr, 610 op/s; 150/1764 objects degraded (8.503%) 2024-04-22T15:20:03.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:03 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:03.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:03 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:03.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:03 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:20:03.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:03 smithi186 ceph-mon[49804]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T15:20:04.103 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:20:04.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 229335 11.13 MB/sec execute 373 sec latency 56.878 ms 2024-04-22T15:20:04.644 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:04 smithi119 ceph-mon[63156]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T15:20:04.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:04 smithi186 ceph-mon[49804]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T15:20:04.963 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:20:04.963 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (5m) 41s ago 14m 14.4M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:20:04.963 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (2m) 41s ago 14m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:20:04.963 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (2m) 2m ago 14m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:20:04.963 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (5m) 41s ago 14m 47.6M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:20:04.963 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (8m) 41s ago 8m 38.5M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:20:04.963 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (8m) 2m ago 8m 14.2M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:20:04.963 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (5m) 41s ago 15m 506M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:20:04.963 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (4m) 2m ago 14m 450M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:20:04.963 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (3m) 41s ago 16m 50.3M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:20:04.963 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (3m) 2m ago 13m 38.9M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (8m) 41s ago 8m 280M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (8m) 2m ago 8m 84.4M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (5m) 41s ago 14m 10.4M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (5m) 2m ago 13m 17.4M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (101s) 41s ago 13m 249M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (51s) 41s ago 12m 12.8M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 starting - - - 1936M 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (11m) 41s ago 11m 376M 1936M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (11m) 2m ago 11m 545M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (11m) 2m ago 11m 409M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (10m) 2m ago 10m 581M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (10m) 2m ago 10m 483M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:20:04.964 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (4m) 41s ago 13m 67.7M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:20:05.459 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:20:05.459 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:20:05.459 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:20:05.459 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 6 2024-04-22T15:20:05.460 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:20:05.461 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:20:05.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 229569 11.12 MB/sec execute 374 sec latency 54.016 ms 2024-04-22T15:20:05.578 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:05 smithi119 ceph-mon[63156]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:05.579 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:05 smithi119 ceph-mon[63156]: from='client.44215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:05.579 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:05 smithi119 ceph-mon[63156]: pgmap v141: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.9 MiB/s wr, 621 op/s; 156/1752 objects degraded (8.904%) 2024-04-22T15:20:05.579 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:05 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/213974605' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:05.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:05 smithi186 ceph-mon[49804]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:05.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:05 smithi186 ceph-mon[49804]: from='client.44215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:05.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:05 smithi186 ceph-mon[49804]: pgmap v141: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.9 MiB/s wr, 621 op/s; 156/1752 objects degraded (8.904%) 2024-04-22T15:20:05.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:05 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/213974605' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:05.875 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:20:05.875 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:20:05.875 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:20:05.875 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:20:05.875 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:20:05.875 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:20:05.875 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:20:05.875 INFO:teuthology.orchestra.run.smithi119.stdout: "crash" 2024-04-22T15:20:05.875 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:20:05.876 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "8/23 daemons upgraded", 2024-04-22T15:20:05.876 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:20:05.876 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:20:05.876 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:20:06.431 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 156/1752 objects degraded (8.904%), 19 pgs degraded 2024-04-22T15:20:06.431 INFO:teuthology.orchestra.run.smithi119.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-22T15:20:06.431 INFO:teuthology.orchestra.run.smithi119.stdout: osd.2 (root=default,host=smithi119) is down 2024-04-22T15:20:06.431 INFO:teuthology.orchestra.run.smithi119.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 156/1752 objects degraded (8.904%), 19 pgs degraded 2024-04-22T15:20:06.431 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-04-22T15:20:06.431 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-04-22T15:20:06.431 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-04-22T15:20:06.431 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-04-22T15:20:06.431 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-04-22T15:20:06.431 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-04-22T15:20:06.432 INFO:teuthology.orchestra.run.smithi119.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-04-22T15:20:06.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 229846 11.11 MB/sec execute 375 sec latency 87.064 ms 2024-04-22T15:20:06.719 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:06 smithi119 ceph-mon[63156]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:06.719 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:06 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/1006306678' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:20:06.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:06 smithi186 ceph-mon[49804]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:06.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:06 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/1006306678' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:20:07.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 230090 11.10 MB/sec execute 376 sec latency 62.255 ms 2024-04-22T15:20:07.732 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:07 smithi119 ceph-mon[63156]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:07.732 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:07 smithi119 ceph-mon[63156]: pgmap v142: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.4 MiB/s wr, 534 op/s; 147/1623 objects degraded (9.057%) 2024-04-22T15:20:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:07 smithi186 ceph-mon[49804]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:07 smithi186 ceph-mon[49804]: pgmap v142: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.4 MiB/s wr, 534 op/s; 147/1623 objects degraded (9.057%) 2024-04-22T15:20:08.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 230357 11.10 MB/sec execute 377 sec latency 61.904 ms 2024-04-22T15:20:08.750 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:08 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2[74475]: 2024-04-22T15:20:08.643+0000 7fd428d757c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-22T15:20:08.751 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:08 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 147/1623 objects degraded (9.057%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:09.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:08 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 147/1623 objects degraded (9.057%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:09.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 230605 11.07 MB/sec execute 378 sec latency 62.469 ms 2024-04-22T15:20:09.769 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:09 smithi119 ceph-mon[63156]: pgmap v143: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 487 op/s; 147/1623 objects degraded (9.057%) 2024-04-22T15:20:09.769 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:09 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2[74475]: 2024-04-22T15:20:09.725+0000 7fd428d757c0 -1 osd.2 98 log_to_monitors true 2024-04-22T15:20:10.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:09 smithi186 ceph-mon[49804]: pgmap v143: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 487 op/s; 147/1623 objects degraded (9.057%) 2024-04-22T15:20:10.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 230982 11.06 MB/sec execute 379 sec latency 67.029 ms 2024-04-22T15:20:10.812 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:10 smithi119 ceph-mon[63156]: from='osd.2 [v2:172.21.15.119:6818/3930344314,v1:172.21.15.119:6819/3930344314]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:20:10.812 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:10 smithi119 ceph-mon[63156]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:20:10.812 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:10.813 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:10.813 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:10 smithi186 ceph-mon[49804]: from='osd.2 [v2:172.21.15.119:6818/3930344314,v1:172.21.15.119:6819/3930344314]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:20:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:10 smithi186 ceph-mon[49804]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:20:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:11.063 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:20:11 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2[74475]: 2024-04-22T15:20:11.052+0000 7fd4161c5700 -1 osd.2 98 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:20:11.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 231169 11.04 MB/sec execute 380 sec latency 103.203 ms 2024-04-22T15:20:11.573 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:11 smithi119 ceph-mon[63156]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T15:20:11.573 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:11 smithi119 ceph-mon[63156]: osdmap e101: 8 total, 7 up, 8 in 2024-04-22T15:20:11.574 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:11 smithi119 ceph-mon[63156]: from='osd.2 [v2:172.21.15.119:6818/3930344314,v1:172.21.15.119:6819/3930344314]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:20:11.574 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:11 smithi119 ceph-mon[63156]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:20:11.574 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:11 smithi119 ceph-mon[63156]: pgmap v145: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 4.9 MiB/s wr, 472 op/s; 150/1629 objects degraded (9.208%) 2024-04-22T15:20:11.574 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:11 smithi119 ceph-mon[63156]: from='osd.2 ' entity='osd.2' 2024-04-22T15:20:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:11 smithi186 ceph-mon[49804]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T15:20:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:11 smithi186 ceph-mon[49804]: osdmap e101: 8 total, 7 up, 8 in 2024-04-22T15:20:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:11 smithi186 ceph-mon[49804]: from='osd.2 [v2:172.21.15.119:6818/3930344314,v1:172.21.15.119:6819/3930344314]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:20:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:11 smithi186 ceph-mon[49804]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:20:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:11 smithi186 ceph-mon[49804]: pgmap v145: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 4.9 MiB/s wr, 472 op/s; 150/1629 objects degraded (9.208%) 2024-04-22T15:20:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:11 smithi186 ceph-mon[49804]: from='osd.2 ' entity='osd.2' 2024-04-22T15:20:12.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 231287 11.02 MB/sec execute 381 sec latency 501.693 ms 2024-04-22T15:20:12.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:12 smithi119 ceph-mon[63156]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:20:12.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:12 smithi119 ceph-mon[63156]: osd.2 [v2:172.21.15.119:6818/3930344314,v1:172.21.15.119:6819/3930344314] boot 2024-04-22T15:20:12.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:12 smithi119 ceph-mon[63156]: osdmap e102: 8 total, 8 up, 8 in 2024-04-22T15:20:12.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:12 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:20:13.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:12 smithi186 ceph-mon[49804]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:20:13.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:12 smithi186 ceph-mon[49804]: osd.2 [v2:172.21.15.119:6818/3930344314,v1:172.21.15.119:6819/3930344314] boot 2024-04-22T15:20:13.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:12 smithi186 ceph-mon[49804]: osdmap e102: 8 total, 8 up, 8 in 2024-04-22T15:20:13.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:12 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:20:13.510 INFO:teuthology.orchestra.run.smithi119.stdout: 5 231419 11.01 MB/sec execute 382 sec latency 1091.015 ms 2024-04-22T15:20:13.593 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:13 smithi119 ceph-mon[63156]: pgmap v147: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.1 MiB/s wr, 394 op/s; 150/1629 objects degraded (9.208%) 2024-04-22T15:20:13.593 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:13 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:13.593 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:13 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:13.593 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:13 smithi119 ceph-mon[63156]: osdmap e103: 8 total, 8 up, 8 in 2024-04-22T15:20:13.594 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:13 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 150/1629 objects degraded (9.208%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:13 smithi186 ceph-mon[49804]: pgmap v147: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.1 MiB/s wr, 394 op/s; 150/1629 objects degraded (9.208%) 2024-04-22T15:20:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:13 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:13 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:13 smithi186 ceph-mon[49804]: osdmap e103: 8 total, 8 up, 8 in 2024-04-22T15:20:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:13 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 150/1629 objects degraded (9.208%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:14.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 231593 11.01 MB/sec execute 383 sec latency 876.665 ms 2024-04-22T15:20:15.150 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:15 smithi119 ceph-mon[63156]: osdmap e104: 8 total, 8 up, 8 in 2024-04-22T15:20:15.150 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:15 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:15.150 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:15 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:15.150 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:15 smithi119 ceph-mon[63156]: pgmap v150: 97 pgs: 3 remapped+peering, 16 active+undersized+degraded, 4 active+undersized, 74 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 474 op/s; 109/1644 objects degraded (6.630%) 2024-04-22T15:20:15.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 231787 10.99 MB/sec execute 384 sec latency 285.173 ms 2024-04-22T15:20:15.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:15 smithi186 ceph-mon[49804]: osdmap e104: 8 total, 8 up, 8 in 2024-04-22T15:20:15.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:15 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:15.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:15 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:15.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:15 smithi186 ceph-mon[49804]: pgmap v150: 97 pgs: 3 remapped+peering, 16 active+undersized+degraded, 4 active+undersized, 74 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 474 op/s; 109/1644 objects degraded (6.630%) 2024-04-22T15:20:16.162 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:16 smithi119 ceph-mon[63156]: osdmap e105: 8 total, 8 up, 8 in 2024-04-22T15:20:16.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 232045 10.97 MB/sec execute 385 sec latency 840.464 ms 2024-04-22T15:20:16.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:16 smithi186 ceph-mon[49804]: osdmap e105: 8 total, 8 up, 8 in 2024-04-22T15:20:17.232 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:17 smithi119 ceph-mon[63156]: osdmap e106: 8 total, 8 up, 8 in 2024-04-22T15:20:17.232 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:17 smithi119 ceph-mon[63156]: pgmap v153: 97 pgs: 3 remapped+peering, 3 peering, 91 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.3 MiB/s wr, 340 op/s; 62 KiB/s, 24 objects/s recovering 2024-04-22T15:20:17.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 232365 10.97 MB/sec execute 386 sec latency 58.913 ms 2024-04-22T15:20:17.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:17 smithi186 ceph-mon[49804]: osdmap e106: 8 total, 8 up, 8 in 2024-04-22T15:20:17.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:17 smithi186 ceph-mon[49804]: pgmap v153: 97 pgs: 3 remapped+peering, 3 peering, 91 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.3 MiB/s wr, 340 op/s; 62 KiB/s, 24 objects/s recovering 2024-04-22T15:20:18.254 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:18 smithi119 ceph-mon[63156]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 109/1644 objects degraded (6.630%), 16 pgs degraded) 2024-04-22T15:20:18.254 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:18 smithi119 ceph-mon[63156]: Cluster is now healthy 2024-04-22T15:20:18.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 232623 10.96 MB/sec execute 387 sec latency 48.850 ms 2024-04-22T15:20:18.524 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:18 smithi186 ceph-mon[49804]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 109/1644 objects degraded (6.630%), 16 pgs degraded) 2024-04-22T15:20:18.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:18 smithi186 ceph-mon[49804]: Cluster is now healthy 2024-04-22T15:20:19.261 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:19 smithi119 ceph-mon[63156]: pgmap v154: 97 pgs: 3 remapped+peering, 3 peering, 91 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.3 MiB/s wr, 239 op/s; 43 KiB/s, 17 objects/s recovering 2024-04-22T15:20:19.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 233054 10.96 MB/sec execute 388 sec latency 33.756 ms 2024-04-22T15:20:19.524 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:19 smithi186 ceph-mon[49804]: pgmap v154: 97 pgs: 3 remapped+peering, 3 peering, 91 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.3 MiB/s wr, 239 op/s; 43 KiB/s, 17 objects/s recovering 2024-04-22T15:20:20.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 233505 10.96 MB/sec execute 389 sec latency 36.982 ms 2024-04-22T15:20:21.314 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:21.314 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:21.314 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:20:21.314 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:20:21.314 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:21.314 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:21.314 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:20:21.314 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:21.315 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:21.315 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:21.315 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T15:20:21.315 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T15:20:21.315 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: Upgrade: osd.3 is safe to restart 2024-04-22T15:20:21.315 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:21 smithi119 ceph-mon[63156]: pgmap v155: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.0 MiB/s wr, 412 op/s; 230 KiB/s, 24 objects/s recovering 2024-04-22T15:20:21.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 233780 10.96 MB/sec execute 390 sec latency 56.829 ms 2024-04-22T15:20:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:20:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:20:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:20:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T15:20:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T15:20:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: Upgrade: osd.3 is safe to restart 2024-04-22T15:20:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:21 smithi186 ceph-mon[49804]: pgmap v155: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.0 MiB/s wr, 412 op/s; 230 KiB/s, 24 objects/s recovering 2024-04-22T15:20:22.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 234125 10.96 MB/sec execute 391 sec latency 50.392 ms 2024-04-22T15:20:23.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 234474 10.94 MB/sec execute 392 sec latency 47.326 ms 2024-04-22T15:20:23.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:23 smithi119 ceph-mon[63156]: pgmap v156: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.9 MiB/s wr, 361 op/s; 193 KiB/s, 20 objects/s recovering 2024-04-22T15:20:23.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:23.834 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:20:23.834 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:20:24.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:23 smithi186 ceph-mon[49804]: pgmap v156: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.9 MiB/s wr, 361 op/s; 193 KiB/s, 20 objects/s recovering 2024-04-22T15:20:24.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:24.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:20:24.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:20:24.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 234802 10.93 MB/sec execute 393 sec latency 41.814 ms 2024-04-22T15:20:24.876 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:24 smithi119 ceph-mon[63156]: Upgrade: Updating osd.3 2024-04-22T15:20:24.876 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:24 smithi119 ceph-mon[63156]: Deploying daemon osd.3 on smithi119 2024-04-22T15:20:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:24 smithi186 ceph-mon[49804]: Upgrade: Updating osd.3 2024-04-22T15:20:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:24 smithi186 ceph-mon[49804]: Deploying daemon osd.3 on smithi119 2024-04-22T15:20:25.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 235111 10.93 MB/sec execute 394 sec latency 38.450 ms 2024-04-22T15:20:25.894 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:25 smithi119 ceph-mon[63156]: pgmap v157: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.7 MiB/s wr, 513 op/s; 159 KiB/s, 16 objects/s recovering 2024-04-22T15:20:25.895 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:25.895 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:25 smithi186 ceph-mon[49804]: pgmap v157: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.7 MiB/s wr, 513 op/s; 159 KiB/s, 16 objects/s recovering 2024-04-22T15:20:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:26.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 235484 10.93 MB/sec execute 395 sec latency 62.183 ms 2024-04-22T15:20:27.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 235878 10.93 MB/sec execute 396 sec latency 49.710 ms 2024-04-22T15:20:27.852 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:27 smithi119 ceph-mon[63156]: pgmap v158: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 464 op/s; 121 KiB/s, 5 objects/s recovering 2024-04-22T15:20:28.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:27 smithi186 ceph-mon[49804]: pgmap v158: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 464 op/s; 121 KiB/s, 5 objects/s recovering 2024-04-22T15:20:28.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 236167 10.92 MB/sec execute 397 sec latency 50.846 ms 2024-04-22T15:20:28.613 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:28 smithi119 systemd[1]: Stopping Ceph osd.3 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:20:29.121 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:29 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[43264]: 2024-04-22T15:20:29.107+0000 7f4a5e096700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:20:29.121 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:29 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[43264]: 2024-04-22T15:20:29.110+0000 7f4a5e096700 -1 osd.3 106 *** Got signal Terminated *** 2024-04-22T15:20:29.121 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:29 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[43264]: 2024-04-22T15:20:29.110+0000 7f4a5e096700 -1 osd.3 106 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:20:29.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 236516 10.93 MB/sec execute 398 sec latency 64.328 ms 2024-04-22T15:20:29.893 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:29 smithi119 ceph-mon[63156]: pgmap v159: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 413 op/s; 108 KiB/s, 5 objects/s recovering 2024-04-22T15:20:29.893 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:29 smithi119 ceph-mon[63156]: osd.3 marked itself down and dead 2024-04-22T15:20:30.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:29 smithi186 ceph-mon[49804]: pgmap v159: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 413 op/s; 108 KiB/s, 5 objects/s recovering 2024-04-22T15:20:30.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:29 smithi186 ceph-mon[49804]: osd.3 marked itself down and dead 2024-04-22T15:20:30.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 236733 10.91 MB/sec execute 399 sec latency 643.541 ms 2024-04-22T15:20:30.712 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:30 smithi119 podman[76580]: 2024-04-22 15:20:30.454287072 +0000 UTC m=+1.783065649 container died 93f8afe5a053639a78b00b3db80ff972d73bf170036c3c795813b2b2603af386 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, maintainer=Guillaume Abrioux , release=754, vcs-type=git, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, version=8, GIT_CLEAN=True, name=centos-stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD) 2024-04-22T15:20:30.963 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:30 smithi119 ceph-mon[63156]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:20:30.964 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:30 smithi119 ceph-mon[63156]: osdmap e107: 8 total, 7 up, 8 in 2024-04-22T15:20:30.964 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:30 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:30.964 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:30 smithi119 ceph-mon[63156]: osdmap e108: 8 total, 7 up, 8 in 2024-04-22T15:20:30.964 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:30 smithi119 podman[76580]: 2024-04-22 15:20:30.826410438 +0000 UTC m=+2.155189024 container cleanup 93f8afe5a053639a78b00b3db80ff972d73bf170036c3c795813b2b2603af386 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3, name=centos-stream, RELEASE=HEAD, GIT_BRANCH=HEAD, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_CLEAN=True, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True) 2024-04-22T15:20:30.964 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:30 smithi119 bash[76580]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3 2024-04-22T15:20:31.215 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:31 smithi119 podman[76595]: 2024-04-22 15:20:31.179347549 +0000 UTC m=+0.725286038 container remove 93f8afe5a053639a78b00b3db80ff972d73bf170036c3c795813b2b2603af386 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3, GIT_BRANCH=HEAD, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, distribution-scope=public, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , version=8) 2024-04-22T15:20:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:30 smithi186 ceph-mon[49804]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:20:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:30 smithi186 ceph-mon[49804]: osdmap e107: 8 total, 7 up, 8 in 2024-04-22T15:20:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:30 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:30 smithi186 ceph-mon[49804]: osdmap e108: 8 total, 7 up, 8 in 2024-04-22T15:20:31.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 236990 10.89 MB/sec execute 400 sec latency 979.223 ms 2024-04-22T15:20:31.726 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:31 smithi119 podman[76677]: 2024-04-22 15:20:31.502911486 +0000 UTC m=+0.028377918 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:20:31.727 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:31 smithi119 podman[76677]: 2024-04-22 15:20:31.715700466 +0000 UTC m=+0.241166924 container create bc728783084fd5b0f3fff797eb45a8fcca82cd0f15310741295afbd3e9a5b9ff (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-22T15:20:31.978 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:31 smithi119 ceph-mon[63156]: pgmap v161: 97 pgs: 22 peering, 7 stale+active+clean, 68 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 561 op/s 2024-04-22T15:20:31.979 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:31 smithi119 ceph-mon[63156]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-22T15:20:32.229 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:32 smithi119 podman[76677]: 2024-04-22 15:20:32.136084092 +0000 UTC m=+0.661550512 container init bc728783084fd5b0f3fff797eb45a8fcca82cd0f15310741295afbd3e9a5b9ff (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:20:32.229 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:32 smithi119 podman[76677]: 2024-04-22 15:20:32.143589841 +0000 UTC m=+0.669056253 container start bc728783084fd5b0f3fff797eb45a8fcca82cd0f15310741295afbd3e9a5b9ff (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:20:32.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:31 smithi186 ceph-mon[49804]: pgmap v161: 97 pgs: 22 peering, 7 stale+active+clean, 68 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 561 op/s 2024-04-22T15:20:32.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:31 smithi186 ceph-mon[49804]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-22T15:20:32.481 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:32 smithi119 podman[76677]: 2024-04-22 15:20:32.303850554 +0000 UTC m=+0.829316967 container attach bc728783084fd5b0f3fff797eb45a8fcca82cd0f15310741295afbd3e9a5b9ff (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-22T15:20:32.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 237383 10.90 MB/sec execute 401 sec latency 56.323 ms 2024-04-22T15:20:32.733 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:32 smithi119 podman[76677]: 2024-04-22 15:20:32.601745774 +0000 UTC m=+1.127212186 container died bc728783084fd5b0f3fff797eb45a8fcca82cd0f15310741295afbd3e9a5b9ff (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-22T15:20:33.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 237902 10.90 MB/sec execute 402 sec latency 35.631 ms 2024-04-22T15:20:33.995 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:33 smithi119 podman[76703]: 2024-04-22 15:20:33.957491899 +0000 UTC m=+1.342324485 container remove bc728783084fd5b0f3fff797eb45a8fcca82cd0f15310741295afbd3e9a5b9ff (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:20:33.996 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:33 smithi119 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-22T15:20:33.996 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:33 smithi119 systemd[1]: Stopped Ceph osd.3 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:20:33.996 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:33 smithi119 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.3.service: Consumed 4min 40.014s CPU time. 2024-04-22T15:20:33.996 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:33 smithi119 ceph-mon[63156]: pgmap v163: 97 pgs: 22 peering, 7 stale+active+clean, 68 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.6 MiB/s wr, 425 op/s 2024-04-22T15:20:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:33 smithi186 ceph-mon[49804]: pgmap v163: 97 pgs: 22 peering, 7 stale+active+clean, 68 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.6 MiB/s wr, 425 op/s 2024-04-22T15:20:34.497 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:34 smithi119 systemd[1]: Starting Ceph osd.3 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:20:34.515 INFO:teuthology.orchestra.run.smithi119.stdout: 5 238366 10.90 MB/sec execute 403 sec latency 45.237 ms 2024-04-22T15:20:34.750 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:34 smithi119 podman[76800]: 2024-04-22 15:20:34.529053449 +0000 UTC m=+0.026973816 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:20:34.750 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:34 smithi119 podman[76800]: 2024-04-22 15:20:34.666071031 +0000 UTC m=+0.163991375 container create 6efe2c956dfb06c0d4e5da05cb11834774c4a7c469cc02e1f3d920d296d07351 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-22T15:20:35.004 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:34 smithi119 ceph-mon[63156]: pgmap v164: 97 pgs: 13 active+undersized+degraded, 22 peering, 6 active+undersized, 56 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.8 MiB/s wr, 322 op/s; 97/1683 objects degraded (5.764%) 2024-04-22T15:20:35.004 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:34 smithi119 ceph-mon[63156]: Health check failed: Degraded data redundancy: 97/1683 objects degraded (5.764%), 13 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:34 smithi186 ceph-mon[49804]: pgmap v164: 97 pgs: 13 active+undersized+degraded, 22 peering, 6 active+undersized, 56 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.8 MiB/s wr, 322 op/s; 97/1683 objects degraded (5.764%) 2024-04-22T15:20:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:34 smithi186 ceph-mon[49804]: Health check failed: Degraded data redundancy: 97/1683 objects degraded (5.764%), 13 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:35.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 238795 10.91 MB/sec execute 404 sec latency 90.005 ms 2024-04-22T15:20:35.758 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:35 smithi119 podman[76800]: 2024-04-22 15:20:35.50710769 +0000 UTC m=+1.005028041 container init 6efe2c956dfb06c0d4e5da05cb11834774c4a7c469cc02e1f3d920d296d07351 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-22T15:20:35.758 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:35 smithi119 podman[76800]: 2024-04-22 15:20:35.514090529 +0000 UTC m=+1.012010880 container start 6efe2c956dfb06c0d4e5da05cb11834774c4a7c469cc02e1f3d920d296d07351 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-22T15:20:35.758 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:35 smithi119 podman[76800]: 2024-04-22 15:20:35.729586874 +0000 UTC m=+1.227507217 container attach 6efe2c956dfb06c0d4e5da05cb11834774c4a7c469cc02e1f3d920d296d07351 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240416, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-22T15:20:36.260 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate[76814]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:20:36.260 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate[76814]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:20:36.260 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 bash[76800]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:20:36.260 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 bash[76800]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 bash[76800]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 bash[76800]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 bash[76800]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 bash[76800]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 bash[76800]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 bash[76800]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 bash[76800]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate[76814]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate[76814]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate[76814]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate[76814]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate[76814]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate[76814]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T15:20:36.261 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate[76814]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-22T15:20:36.262 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:36 smithi119 podman[76800]: 2024-04-22 15:20:36.240286136 +0000 UTC m=+1.738206493 container died 6efe2c956dfb06c0d4e5da05cb11834774c4a7c469cc02e1f3d920d296d07351 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-22T15:20:36.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 239282 10.91 MB/sec execute 405 sec latency 36.785 ms 2024-04-22T15:20:37.010 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:20:37.016 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:36 smithi119 ceph-mon[63156]: pgmap v165: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.8 MiB/s wr, 669 op/s; 222/1779 objects degraded (12.479%) 2024-04-22T15:20:37.016 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:36 smithi119 ceph-mon[63156]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-04-22T15:20:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:36 smithi186 ceph-mon[49804]: pgmap v165: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.8 MiB/s wr, 669 op/s; 222/1779 objects degraded (12.479%) 2024-04-22T15:20:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:36 smithi186 ceph-mon[49804]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-04-22T15:20:37.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 239651 10.91 MB/sec execute 406 sec latency 40.421 ms 2024-04-22T15:20:37.544 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:37 smithi119 podman[76942]: 2024-04-22 15:20:37.441407492 +0000 UTC m=+1.187441580 container remove 6efe2c956dfb06c0d4e5da05cb11834774c4a7c469cc02e1f3d920d296d07351 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-activate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True) 2024-04-22T15:20:37.800 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:37 smithi119 podman[77052]: 2024-04-22 15:20:37.617867039 +0000 UTC m=+0.025750963 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:20:37.800 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:37 smithi119 podman[77052]: 2024-04-22 15:20:37.742554376 +0000 UTC m=+0.150438299 container create 2378efa46eca2bdca3f039224d119f028d03df30828466aa811fd0e580497f97 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:20:38.053 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:37 smithi119 ceph-mon[63156]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:38.054 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:37 smithi119 ceph-mon[63156]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (6m) 25s ago 15m 14.6M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (2m) 25s ago 15m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (2m) 2m ago 14m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (6m) 25s ago 14m 47.8M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (9m) 25s ago 9m 38.4M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (9m) 2m ago 9m 14.2M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (5m) 25s ago 16m 508M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (4m) 2m ago 14m 450M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (4m) 25s ago 16m 51.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (3m) 2m ago 14m 38.9M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (8m) 25s ago 8m 285M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:20:38.072 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (8m) 2m ago 8m 84.4M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:20:38.073 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (6m) 25s ago 14m 11.0M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:20:38.073 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (5m) 2m ago 14m 17.4M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:20:38.073 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (2m) 25s ago 13m 319M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:20:38.073 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (84s) 25s ago 13m 254M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:20:38.073 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (35s) 25s ago 12m 15.0M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:20:38.073 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (12m) 25s ago 12m 407M 1936M 17.2.0 e1d6a67b021e 93f8afe5a053 2024-04-22T15:20:38.073 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (12m) 2m ago 12m 545M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:20:38.073 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (11m) 2m ago 11m 409M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:20:38.073 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (11m) 2m ago 11m 581M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:20:38.073 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (10m) 2m ago 10m 483M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:20:38.073 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (4m) 25s ago 14m 67.9M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:20:38.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:37 smithi186 ceph-mon[49804]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:38.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:37 smithi186 ceph-mon[49804]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:38.305 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:38 smithi119 podman[77052]: 2024-04-22 15:20:38.233141566 +0000 UTC m=+0.641025495 container init 2378efa46eca2bdca3f039224d119f028d03df30828466aa811fd0e580497f97 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-22T15:20:38.305 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:38 smithi119 podman[77052]: 2024-04-22 15:20:38.245362483 +0000 UTC m=+0.653246414 container start 2378efa46eca2bdca3f039224d119f028d03df30828466aa811fd0e580497f97 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux ) 2024-04-22T15:20:38.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 239951 10.91 MB/sec execute 407 sec latency 36.058 ms 2024-04-22T15:20:38.556 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:38 smithi119 bash[77052]: 2378efa46eca2bdca3f039224d119f028d03df30828466aa811fd0e580497f97 2024-04-22T15:20:38.556 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:38 smithi119 systemd[1]: Started Ceph osd.3 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:20:38.724 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:20:38.724 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:20:38.724 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:20:38.724 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:20:38.724 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:20:38.724 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 7 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:20:38.725 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:20:39.058 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:38 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[77086]: 2024-04-22T15:20:38.838+0000 7f5dcb44f7c0 -1 Falling back to public interface 2024-04-22T15:20:39.301 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:20:39.301 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:20:39.301 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:20:39.301 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:20:39.301 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:20:39.301 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:20:39.301 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:20:39.302 INFO:teuthology.orchestra.run.smithi119.stdout: "crash" 2024-04-22T15:20:39.302 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:20:39.302 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "9/23 daemons upgraded", 2024-04-22T15:20:39.302 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:20:39.302 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:20:39.302 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:20:39.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 240312 10.90 MB/sec execute 408 sec latency 61.125 ms 2024-04-22T15:20:39.561 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:39 smithi119 ceph-mon[63156]: from='client.44243 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:39.562 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:39.562 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:39.562 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:20:39.562 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:39 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/404097773' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:39.562 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:39 smithi119 ceph-mon[63156]: pgmap v166: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.4 MiB/s wr, 562 op/s; 222/1779 objects degraded (12.479%) 2024-04-22T15:20:39.861 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 222/1779 objects degraded (12.479%), 27 pgs degraded 2024-04-22T15:20:39.861 INFO:teuthology.orchestra.run.smithi119.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-22T15:20:39.861 INFO:teuthology.orchestra.run.smithi119.stdout: osd.3 (root=default,host=smithi119) is down 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 222/1779 objects degraded (12.479%), 27 pgs degraded 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-04-22T15:20:39.862 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-04-22T15:20:39.863 INFO:teuthology.orchestra.run.smithi119.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-04-22T15:20:39.864 INFO:teuthology.orchestra.run.smithi119.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-04-22T15:20:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:39 smithi186 ceph-mon[49804]: from='client.44243 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:40.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:39 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:40.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:39 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:40.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:39 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:20:40.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:39 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/404097773' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:40.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:39 smithi186 ceph-mon[49804]: pgmap v166: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.4 MiB/s wr, 562 op/s; 222/1779 objects degraded (12.479%) 2024-04-22T15:20:40.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 240684 10.90 MB/sec execute 409 sec latency 52.514 ms 2024-04-22T15:20:40.838 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:40 smithi119 ceph-mon[63156]: from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:40.838 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:40 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:40.838 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:40 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:40.838 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:40 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/3938786307' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:20:41.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:40 smithi186 ceph-mon[49804]: from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:20:41.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:40 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:41.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:40 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:41.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:40 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/3938786307' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:20:41.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 241043 10.91 MB/sec execute 410 sec latency 42.206 ms 2024-04-22T15:20:41.861 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:41 smithi119 ceph-mon[63156]: pgmap v167: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.7 MiB/s wr, 576 op/s; 212/1686 objects degraded (12.574%) 2024-04-22T15:20:41.861 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:41 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 212/1686 objects degraded (12.574%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:42.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:41 smithi186 ceph-mon[49804]: pgmap v167: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.7 MiB/s wr, 576 op/s; 212/1686 objects degraded (12.574%) 2024-04-22T15:20:42.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:41 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 212/1686 objects degraded (12.574%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:42.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 241466 10.90 MB/sec execute 411 sec latency 51.474 ms 2024-04-22T15:20:43.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 241858 10.89 MB/sec execute 412 sec latency 38.578 ms 2024-04-22T15:20:43.922 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:43 smithi119 ceph-mon[63156]: pgmap v168: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 482 op/s; 212/1686 objects degraded (12.574%) 2024-04-22T15:20:44.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:43 smithi186 ceph-mon[49804]: pgmap v168: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 482 op/s; 212/1686 objects degraded (12.574%) 2024-04-22T15:20:44.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 242114 10.88 MB/sec execute 413 sec latency 43.398 ms 2024-04-22T15:20:45.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 242496 10.89 MB/sec execute 414 sec latency 99.255 ms 2024-04-22T15:20:45.933 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:45 smithi119 ceph-mon[63156]: pgmap v169: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.8 MiB/s wr, 573 op/s; 228/1779 objects degraded (12.816%) 2024-04-22T15:20:45.934 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:45 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[77086]: 2024-04-22T15:20:45.683+0000 7f5dcb44f7c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-22T15:20:46.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:45 smithi186 ceph-mon[49804]: pgmap v169: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.8 MiB/s wr, 573 op/s; 228/1779 objects degraded (12.816%) 2024-04-22T15:20:46.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 242768 10.87 MB/sec execute 415 sec latency 79.701 ms 2024-04-22T15:20:46.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:46 smithi119 ceph-mon[63156]: pgmap v170: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 648 op/s; 211/1668 objects degraded (12.650%) 2024-04-22T15:20:46.952 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:46 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 211/1668 objects degraded (12.650%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:47.203 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:47 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[77086]: 2024-04-22T15:20:47.114+0000 7f5dcb44f7c0 -1 osd.3 106 log_to_monitors true 2024-04-22T15:20:47.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:46 smithi186 ceph-mon[49804]: pgmap v170: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 648 op/s; 211/1668 objects degraded (12.650%) 2024-04-22T15:20:47.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:46 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 211/1668 objects degraded (12.650%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:47.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 243072 10.87 MB/sec execute 416 sec latency 67.660 ms 2024-04-22T15:20:47.970 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:47 smithi119 ceph-mon[63156]: from='osd.3 [v2:172.21.15.119:6826/1077847878,v1:172.21.15.119:6827/1077847878]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:20:47.970 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:20:47 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[77086]: 2024-04-22T15:20:47.894+0000 7f5dbc8a7700 -1 osd.3 106 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:20:48.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:47 smithi186 ceph-mon[49804]: from='osd.3 [v2:172.21.15.119:6826/1077847878,v1:172.21.15.119:6827/1077847878]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:20:48.516 INFO:teuthology.orchestra.run.smithi119.stdout: 5 243247 10.86 MB/sec execute 417 sec latency 48.801 ms 2024-04-22T15:20:49.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:48 smithi119 ceph-mon[63156]: from='osd.3 [v2:172.21.15.119:6826/1077847878,v1:172.21.15.119:6827/1077847878]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T15:20:49.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:48 smithi119 ceph-mon[63156]: osdmap e109: 8 total, 7 up, 8 in 2024-04-22T15:20:49.026 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:48 smithi119 ceph-mon[63156]: from='osd.3 [v2:172.21.15.119:6826/1077847878,v1:172.21.15.119:6827/1077847878]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:20:49.026 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:49.026 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:49.026 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:48 smithi119 ceph-mon[63156]: pgmap v172: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.1 MiB/s wr, 499 op/s; 211/1668 objects degraded (12.650%) 2024-04-22T15:20:49.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:48 smithi186 ceph-mon[49804]: from='osd.3 [v2:172.21.15.119:6826/1077847878,v1:172.21.15.119:6827/1077847878]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T15:20:49.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:48 smithi186 ceph-mon[49804]: osdmap e109: 8 total, 7 up, 8 in 2024-04-22T15:20:49.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:48 smithi186 ceph-mon[49804]: from='osd.3 [v2:172.21.15.119:6826/1077847878,v1:172.21.15.119:6827/1077847878]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi119", "root=default"]}]: dispatch 2024-04-22T15:20:49.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:49.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:49.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:48 smithi186 ceph-mon[49804]: pgmap v172: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.1 MiB/s wr, 499 op/s; 211/1668 objects degraded (12.650%) 2024-04-22T15:20:49.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 243386 10.84 MB/sec execute 418 sec latency 617.591 ms 2024-04-22T15:20:50.135 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:49 smithi119 ceph-mon[63156]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:20:50.135 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:49 smithi119 ceph-mon[63156]: osd.3 [v2:172.21.15.119:6826/1077847878,v1:172.21.15.119:6827/1077847878] boot 2024-04-22T15:20:50.135 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:49 smithi119 ceph-mon[63156]: osdmap e110: 8 total, 8 up, 8 in 2024-04-22T15:20:50.136 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:49 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:20:50.136 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:49 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:50.136 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:49 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:50.136 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:49 smithi119 ceph-mon[63156]: osdmap e111: 8 total, 8 up, 8 in 2024-04-22T15:20:50.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:49 smithi186 ceph-mon[49804]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:20:50.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:49 smithi186 ceph-mon[49804]: osd.3 [v2:172.21.15.119:6826/1077847878,v1:172.21.15.119:6827/1077847878] boot 2024-04-22T15:20:50.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:49 smithi186 ceph-mon[49804]: osdmap e110: 8 total, 8 up, 8 in 2024-04-22T15:20:50.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:49 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:20:50.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:49 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:50.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:49 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:50.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:49 smithi186 ceph-mon[49804]: osdmap e111: 8 total, 8 up, 8 in 2024-04-22T15:20:50.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 243482 10.82 MB/sec execute 419 sec latency 1539.794 ms 2024-04-22T15:20:50.935 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:50 smithi119 ceph-mon[63156]: pgmap v175: 97 pgs: 5 remapped+peering, 1 active+recovering+degraded, 6 peering, 14 active+undersized+degraded, 6 active+undersized, 65 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 557 op/s; 107/1596 objects degraded (6.704%); 14 B/s, 3 objects/s recovering 2024-04-22T15:20:50.935 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:50 smithi119 ceph-mon[63156]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T15:20:51.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:50 smithi186 ceph-mon[49804]: pgmap v175: 97 pgs: 5 remapped+peering, 1 active+recovering+degraded, 6 peering, 14 active+undersized+degraded, 6 active+undersized, 65 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 557 op/s; 107/1596 objects degraded (6.704%); 14 B/s, 3 objects/s recovering 2024-04-22T15:20:51.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:50 smithi186 ceph-mon[49804]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T15:20:51.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 243683 10.80 MB/sec execute 420 sec latency 1971.402 ms 2024-04-22T15:20:52.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 244073 10.80 MB/sec execute 421 sec latency 42.853 ms 2024-04-22T15:20:52.968 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:52 smithi119 ceph-mon[63156]: osdmap e113: 8 total, 8 up, 8 in 2024-04-22T15:20:52.968 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:52 smithi119 ceph-mon[63156]: pgmap v178: 97 pgs: 5 remapped+peering, 1 active+recovering+degraded, 6 peering, 14 active+undersized+degraded, 6 active+undersized, 65 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 529 op/s; 107/1596 objects degraded (6.704%); 22 B/s, 4 objects/s recovering 2024-04-22T15:20:53.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:52 smithi186 ceph-mon[49804]: osdmap e113: 8 total, 8 up, 8 in 2024-04-22T15:20:53.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:52 smithi186 ceph-mon[49804]: pgmap v178: 97 pgs: 5 remapped+peering, 1 active+recovering+degraded, 6 peering, 14 active+undersized+degraded, 6 active+undersized, 65 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 529 op/s; 107/1596 objects degraded (6.704%); 22 B/s, 4 objects/s recovering 2024-04-22T15:20:53.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 244427 10.80 MB/sec execute 422 sec latency 243.339 ms 2024-04-22T15:20:53.990 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:53 smithi119 ceph-mon[63156]: osdmap e114: 8 total, 8 up, 8 in 2024-04-22T15:20:53.990 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:53 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 107/1596 objects degraded (6.704%), 15 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:53.990 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:53 smithi119 ceph-mon[63156]: osdmap e115: 8 total, 8 up, 8 in 2024-04-22T15:20:54.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:53 smithi186 ceph-mon[49804]: osdmap e114: 8 total, 8 up, 8 in 2024-04-22T15:20:54.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:53 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 107/1596 objects degraded (6.704%), 15 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:54.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:53 smithi186 ceph-mon[49804]: osdmap e115: 8 total, 8 up, 8 in 2024-04-22T15:20:54.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 244653 10.79 MB/sec execute 423 sec latency 927.074 ms 2024-04-22T15:20:55.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 244904 10.78 MB/sec execute 424 sec latency 930.730 ms 2024-04-22T15:20:55.669 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:55 smithi119 ceph-mon[63156]: osdmap e116: 8 total, 8 up, 8 in 2024-04-22T15:20:55.669 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:55 smithi119 ceph-mon[63156]: pgmap v182: 97 pgs: 85 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 3 peering, 1 active+recovering+degraded, 4 remapped+peering; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 530 B/s rd, 2.0 MiB/s wr, 295 op/s; 87/1632 objects degraded (5.331%); 1.5 MiB/s, 30 objects/s recovering 2024-04-22T15:20:55.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:55.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:55.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:55.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:55.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:20:55.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:20:55.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:55.670 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:55.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:55 smithi186 ceph-mon[49804]: osdmap e116: 8 total, 8 up, 8 in 2024-04-22T15:20:55.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:55 smithi186 ceph-mon[49804]: pgmap v182: 97 pgs: 85 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 3 peering, 1 active+recovering+degraded, 4 remapped+peering; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 530 B/s rd, 2.0 MiB/s wr, 295 op/s; 87/1632 objects degraded (5.331%); 1.5 MiB/s, 30 objects/s recovering 2024-04-22T15:20:55.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:55.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:55.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:55.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:55.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:20:55.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:20:55.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:55.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:20:56.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 245220 10.77 MB/sec execute 425 sec latency 812.692 ms 2024-04-22T15:20:56.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:56 smithi119 ceph-mon[63156]: osdmap e117: 8 total, 8 up, 8 in 2024-04-22T15:20:56.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:56 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:20:56.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:56 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:56.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:56 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:56.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:56 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:56.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:56 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:20:56.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:56 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:20:56.684 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:56 smithi119 ceph-mon[63156]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-22T15:20:56.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:56 smithi186 ceph-mon[49804]: osdmap e117: 8 total, 8 up, 8 in 2024-04-22T15:20:56.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:56 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:20:56.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:56 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:56.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:56 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:56.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:56 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:20:56.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:56 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:20:56.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:56 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:20:56.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:56 smithi186 ceph-mon[49804]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-22T15:20:57.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 245671 10.78 MB/sec execute 426 sec latency 821.643 ms 2024-04-22T15:20:57.691 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:57 smithi119 ceph-mon[63156]: osdmap e118: 8 total, 8 up, 8 in 2024-04-22T15:20:57.692 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:57 smithi119 ceph-mon[63156]: pgmap v185: 97 pgs: 89 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 3 peering, 1 remapped+peering; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 454 op/s; 87/1674 objects degraded (5.197%); 2.5 MiB/s, 34 objects/s recovering 2024-04-22T15:20:57.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:57 smithi186 ceph-mon[49804]: osdmap e118: 8 total, 8 up, 8 in 2024-04-22T15:20:57.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:57 smithi186 ceph-mon[49804]: pgmap v185: 97 pgs: 89 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 3 peering, 1 remapped+peering; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 454 op/s; 87/1674 objects degraded (5.197%); 2.5 MiB/s, 34 objects/s recovering 2024-04-22T15:20:58.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 246101 10.79 MB/sec execute 427 sec latency 38.252 ms 2024-04-22T15:20:58.721 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:58 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 87/1674 objects degraded (5.197%), 4 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:58.774 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:58 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 87/1674 objects degraded (5.197%), 4 pgs degraded (PG_DEGRADED) 2024-04-22T15:20:59.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 246482 10.78 MB/sec execute 428 sec latency 41.277 ms 2024-04-22T15:20:59.736 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:20:59 smithi119 ceph-mon[63156]: pgmap v186: 97 pgs: 95 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.3 MiB/s wr, 374 op/s; 21/1680 objects degraded (1.250%); 2.6 MiB/s, 45 objects/s recovering 2024-04-22T15:20:59.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:20:59 smithi186 ceph-mon[49804]: pgmap v186: 97 pgs: 95 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.3 MiB/s wr, 374 op/s; 21/1680 objects degraded (1.250%); 2.6 MiB/s, 45 objects/s recovering 2024-04-22T15:21:00.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 246814 10.77 MB/sec execute 429 sec latency 41.408 ms 2024-04-22T15:21:00.980 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:00 smithi119 ceph-mon[63156]: pgmap v187: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 572 op/s; 1.4 MiB/s, 29 objects/s recovering 2024-04-22T15:21:00.980 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:00 smithi119 ceph-mon[63156]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/1680 objects degraded (1.250%), 1 pg degraded) 2024-04-22T15:21:00.981 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:00 smithi119 ceph-mon[63156]: Cluster is now healthy 2024-04-22T15:21:01.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:00 smithi186 ceph-mon[49804]: pgmap v187: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 572 op/s; 1.4 MiB/s, 29 objects/s recovering 2024-04-22T15:21:01.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:00 smithi186 ceph-mon[49804]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/1680 objects degraded (1.250%), 1 pg degraded) 2024-04-22T15:21:01.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:00 smithi186 ceph-mon[49804]: Cluster is now healthy 2024-04-22T15:21:01.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 247105 10.77 MB/sec execute 430 sec latency 45.750 ms 2024-04-22T15:21:02.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 247476 10.77 MB/sec execute 431 sec latency 53.530 ms 2024-04-22T15:21:03.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 247832 10.77 MB/sec execute 432 sec latency 45.081 ms 2024-04-22T15:21:03.796 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:03 smithi119 ceph-mon[63156]: pgmap v188: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 446 op/s; 1.1 MiB/s, 22 objects/s recovering 2024-04-22T15:21:04.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:03 smithi186 ceph-mon[49804]: pgmap v188: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 446 op/s; 1.1 MiB/s, 22 objects/s recovering 2024-04-22T15:21:04.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 248111 10.76 MB/sec execute 433 sec latency 65.838 ms 2024-04-22T15:21:05.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 248420 10.76 MB/sec execute 434 sec latency 46.952 ms 2024-04-22T15:21:05.831 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:05 smithi119 ceph-mon[63156]: pgmap v189: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 440 op/s; 1002 KiB/s, 19 objects/s recovering 2024-04-22T15:21:06.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:05 smithi186 ceph-mon[49804]: pgmap v189: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 440 op/s; 1002 KiB/s, 19 objects/s recovering 2024-04-22T15:21:06.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 248655 10.74 MB/sec execute 435 sec latency 79.627 ms 2024-04-22T15:21:07.518 INFO:teuthology.orchestra.run.smithi119.stdout: 5 248976 10.74 MB/sec execute 436 sec latency 55.081 ms 2024-04-22T15:21:07.864 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:07 smithi119 ceph-mon[63156]: pgmap v190: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 536 op/s; 504 KiB/s, 12 objects/s recovering 2024-04-22T15:21:08.076 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:07 smithi186 ceph-mon[49804]: pgmap v190: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 536 op/s; 504 KiB/s, 12 objects/s recovering 2024-04-22T15:21:08.518 INFO:teuthology.orchestra.run.smithi119.stdout: 5 249212 10.72 MB/sec execute 437 sec latency 47.167 ms 2024-04-22T15:21:09.517 INFO:teuthology.orchestra.run.smithi119.stdout: 5 249518 10.73 MB/sec execute 438 sec latency 47.320 ms 2024-04-22T15:21:09.916 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:09 smithi119 ceph-mon[63156]: pgmap v191: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.9 MiB/s wr, 468 op/s; 430 KiB/s, 10 objects/s recovering 2024-04-22T15:21:09.917 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:09.917 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:10.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:09 smithi186 ceph-mon[49804]: pgmap v191: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.9 MiB/s wr, 468 op/s; 430 KiB/s, 10 objects/s recovering 2024-04-22T15:21:10.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:10.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:10.299 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:21:10.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 249824 10.72 MB/sec execute 439 sec latency 76.536 ms 2024-04-22T15:21:10.944 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:10.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:10 smithi119 ceph-mon[63156]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:10.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:21:10.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:10 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:21:10.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:10 smithi119 ceph-mon[63156]: Upgrade: osd.4 is safe to restart 2024-04-22T15:21:10.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:10 smithi119 ceph-mon[63156]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:10.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:10 smithi119 ceph-mon[63156]: pgmap v192: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.8 MiB/s wr, 614 op/s; 59 KiB/s, 1 objects/s recovering 2024-04-22T15:21:11.133 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:21:11.133 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (6m) 23s ago 15m 15.0M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:21:11.133 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (3m) 23s ago 15m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:21:11.133 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (3m) 3m ago 15m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (6m) 23s ago 15m 47.8M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (9m) 23s ago 9m 38.4M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (9m) 3m ago 9m 14.2M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (6m) 23s ago 17m 509M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (5m) 3m ago 15m 450M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (4m) 23s ago 17m 52.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (4m) 3m ago 15m 38.9M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (9m) 23s ago 9m 285M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (9m) 3m ago 9m 84.4M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (6m) 23s ago 15m 10.4M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (6m) 3m ago 14m 17.4M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (2m) 23s ago 14m 357M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (117s) 23s ago 13m 286M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (68s) 23s ago 13m 220M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:21:11.134 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (33s) 23s ago 12m 12.8M 1936M 19.0.0-3223-g5883d494 3d983c26e278 2378efa46eca 2024-04-22T15:21:11.135 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (12m) 3m ago 12m 545M 2704M 17.2.0 e1d6a67b021e d0dda3022905 2024-04-22T15:21:11.135 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (12m) 3m ago 12m 409M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:21:11.135 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (11m) 3m ago 11m 581M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:21:11.135 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (11m) 3m ago 11m 483M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:21:11.135 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (5m) 23s ago 14m 68.0M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:21:11.257 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:11.258 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:10 smithi186 ceph-mon[49804]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:11.258 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:21:11.258 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:10 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:21:11.258 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:10 smithi186 ceph-mon[49804]: Upgrade: osd.4 is safe to restart 2024-04-22T15:21:11.258 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:10 smithi186 ceph-mon[49804]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:11.258 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:10 smithi186 ceph-mon[49804]: pgmap v192: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.8 MiB/s wr, 614 op/s; 59 KiB/s, 1 objects/s recovering 2024-04-22T15:21:11.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 250166 10.71 MB/sec execute 440 sec latency 47.973 ms 2024-04-22T15:21:11.632 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 4 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:21:11.633 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:21:11.634 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-22T15:21:11.634 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:21:11.634 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:21:11.634 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:21:11.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:11 smithi119 ceph-mon[63156]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:11.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:11 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/3957450892' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:21:12.056 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:21:12.056 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:21:12.056 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:21:12.056 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:21:12.057 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:21:12.057 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:21:12.057 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:21:12.057 INFO:teuthology.orchestra.run.smithi119.stdout: "crash" 2024-04-22T15:21:12.057 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:21:12.057 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "10/23 daemons upgraded", 2024-04-22T15:21:12.057 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:21:12.057 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:21:12.058 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:21:12.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:11 smithi186 ceph-mon[49804]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:12.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:11 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/3957450892' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:21:12.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 250401 10.70 MB/sec execute 441 sec latency 54.395 ms 2024-04-22T15:21:12.600 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:21:12.999 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:12 smithi119 ceph-mon[63156]: from='client.34354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:12.999 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:12 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/2715137311' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:21:13.000 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:12 smithi119 ceph-mon[63156]: pgmap v193: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 387 op/s 2024-04-22T15:21:13.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:12 smithi186 ceph-mon[49804]: from='client.34354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:13.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:12 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/2715137311' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:21:13.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:12 smithi186 ceph-mon[49804]: pgmap v193: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 387 op/s 2024-04-22T15:21:13.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 250641 10.69 MB/sec execute 442 sec latency 78.848 ms 2024-04-22T15:21:14.033 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:14 smithi119 ceph-mon[63156]: Upgrade: Updating osd.4 2024-04-22T15:21:14.034 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:14 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:14.034 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:14 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:21:14.034 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:14 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:14.034 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:14 smithi119 ceph-mon[63156]: Deploying daemon osd.4 on smithi186 2024-04-22T15:21:14.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:14 smithi186 ceph-mon[49804]: Upgrade: Updating osd.4 2024-04-22T15:21:14.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:14 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:14.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:14 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:21:14.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:14 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:14.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:14 smithi186 ceph-mon[49804]: Deploying daemon osd.4 on smithi186 2024-04-22T15:21:14.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 251002 10.69 MB/sec execute 443 sec latency 48.824 ms 2024-04-22T15:21:15.065 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:15 smithi119 ceph-mon[63156]: pgmap v194: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.5 MiB/s wr, 418 op/s 2024-04-22T15:21:15.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:15 smithi186 ceph-mon[49804]: pgmap v194: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.5 MiB/s wr, 418 op/s 2024-04-22T15:21:15.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 251355 10.69 MB/sec execute 444 sec latency 42.338 ms 2024-04-22T15:21:16.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 251601 10.68 MB/sec execute 445 sec latency 57.131 ms 2024-04-22T15:21:17.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 251945 10.68 MB/sec execute 446 sec latency 42.628 ms 2024-04-22T15:21:17.525 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:17 smithi186 systemd[1]: Stopping Ceph osd.4 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:21:17.800 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:17 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[28181]: 2024-04-22T15:21:17.529+0000 7fe7f6f21700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:21:17.800 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:17 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[28181]: 2024-04-22T15:21:17.529+0000 7fe7f6f21700 -1 osd.4 118 *** Got signal Terminated *** 2024-04-22T15:21:17.800 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:17 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[28181]: 2024-04-22T15:21:17.529+0000 7fe7f6f21700 -1 osd.4 118 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:21:17.916 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:17 smithi119 ceph-mon[63156]: pgmap v195: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.7 MiB/s wr, 528 op/s 2024-04-22T15:21:17.916 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:17 smithi119 ceph-mon[63156]: osd.4 marked itself down and dead 2024-04-22T15:21:18.055 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:17 smithi186 ceph-mon[49804]: pgmap v195: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.7 MiB/s wr, 528 op/s 2024-04-22T15:21:18.055 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:17 smithi186 ceph-mon[49804]: osd.4 marked itself down and dead 2024-04-22T15:21:18.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 252174 10.66 MB/sec execute 447 sec latency 697.243 ms 2024-04-22T15:21:18.633 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:18 smithi186 podman[54286]: 2024-04-22 15:21:18.314104037 +0000 UTC m=+0.902525938 container died d0dda3022905c5414d513f15499ea7eb8a806ff45b99cda41434381a9765d96c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, release=754, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, ceph=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, version=8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T15:21:18.958 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:18 smithi186 ceph-mon[49804]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:21:18.959 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:18 smithi186 ceph-mon[49804]: osdmap e119: 8 total, 7 up, 8 in 2024-04-22T15:21:18.959 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:18 smithi186 podman[54286]: 2024-04-22 15:21:18.73002024 +0000 UTC m=+1.318442136 container cleanup d0dda3022905c5414d513f15499ea7eb8a806ff45b99cda41434381a9765d96c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.buildah.version=1.19.8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, GIT_BRANCH=HEAD) 2024-04-22T15:21:18.959 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:18 smithi186 bash[54286]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4 2024-04-22T15:21:19.061 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:18 smithi119 ceph-mon[63156]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:21:19.061 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:18 smithi119 ceph-mon[63156]: osdmap e119: 8 total, 7 up, 8 in 2024-04-22T15:21:19.275 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:18 smithi186 podman[54305]: 2024-04-22 15:21:18.956317134 +0000 UTC m=+0.634684247 container remove d0dda3022905c5414d513f15499ea7eb8a806ff45b99cda41434381a9765d96c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4, distribution-scope=public, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, vendor=Red Hat, Inc., io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-22T15:21:19.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 252345 10.64 MB/sec execute 448 sec latency 1017.254 ms 2024-04-22T15:21:19.579 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:19 smithi186 podman[54429]: 2024-04-22 15:21:19.203550267 +0000 UTC m=+0.019325261 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:21:19.579 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:19 smithi186 podman[54429]: 2024-04-22 15:21:19.332729662 +0000 UTC m=+0.148504640 container create 94351ecedc6492f4470caeb366573eaf9edbbc3ad4cb2884cbc56334995012b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-22T15:21:19.580 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:19 smithi186 podman[54429]: 2024-04-22 15:21:19.577240331 +0000 UTC m=+0.393015297 container init 94351ecedc6492f4470caeb366573eaf9edbbc3ad4cb2884cbc56334995012b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-22T15:21:19.850 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:19 smithi119 ceph-mon[63156]: pgmap v197: 97 pgs: 5 peering, 11 stale+active+clean, 81 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 430 op/s 2024-04-22T15:21:19.850 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:19 smithi119 ceph-mon[63156]: osdmap e120: 8 total, 7 up, 8 in 2024-04-22T15:21:19.906 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:19 smithi186 ceph-mon[49804]: pgmap v197: 97 pgs: 5 peering, 11 stale+active+clean, 81 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 430 op/s 2024-04-22T15:21:19.907 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:19 smithi186 ceph-mon[49804]: osdmap e120: 8 total, 7 up, 8 in 2024-04-22T15:21:19.907 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:19 smithi186 podman[54429]: 2024-04-22 15:21:19.58054351 +0000 UTC m=+0.396318486 container start 94351ecedc6492f4470caeb366573eaf9edbbc3ad4cb2884cbc56334995012b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:21:19.907 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:19 smithi186 podman[54429]: 2024-04-22 15:21:19.685707938 +0000 UTC m=+0.501482917 container attach 94351ecedc6492f4470caeb366573eaf9edbbc3ad4cb2884cbc56334995012b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-22T15:21:20.216 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:19 smithi186 podman[54429]: 2024-04-22 15:21:19.904467178 +0000 UTC m=+0.720242158 container died 94351ecedc6492f4470caeb366573eaf9edbbc3ad4cb2884cbc56334995012b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True) 2024-04-22T15:21:20.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 252635 10.64 MB/sec execute 449 sec latency 63.182 ms 2024-04-22T15:21:20.859 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:20 smithi186 podman[54455]: 2024-04-22 15:21:20.5318322 +0000 UTC m=+0.622108863 container remove 94351ecedc6492f4470caeb366573eaf9edbbc3ad4cb2884cbc56334995012b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-22T15:21:20.859 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:20 smithi186 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-22T15:21:20.859 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:20 smithi186 systemd[1]: Stopped Ceph osd.4 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:21:20.859 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:20 smithi186 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.4.service: Consumed 4min 41.457s CPU time. 2024-04-22T15:21:20.859 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:20 smithi186 systemd[1]: Starting Ceph osd.4 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:21:20.888 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:20 smithi119 ceph-mon[63156]: pgmap v199: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 5 peering, 56 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.0 MiB/s wr, 519 op/s; 246/1632 objects degraded (15.074%) 2024-04-22T15:21:20.889 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:20 smithi119 ceph-mon[63156]: Health check failed: Degraded data redundancy: 246/1632 objects degraded (15.074%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T15:21:21.137 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:20 smithi186 ceph-mon[49804]: pgmap v199: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 5 peering, 56 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.0 MiB/s wr, 519 op/s; 246/1632 objects degraded (15.074%) 2024-04-22T15:21:21.137 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:20 smithi186 ceph-mon[49804]: Health check failed: Degraded data redundancy: 246/1632 objects degraded (15.074%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T15:21:21.137 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:20 smithi186 podman[54560]: 2024-04-22 15:21:20.884167403 +0000 UTC m=+0.018329868 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:21:21.137 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 podman[54560]: 2024-04-22 15:21:21.016632559 +0000 UTC m=+0.150795031 container create eb4af56c72691747ed85521c46111577d093ed98210ad1df6a56cbec99ecc8ea (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:21:21.413 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 podman[54560]: 2024-04-22 15:21:21.303166039 +0000 UTC m=+0.437328484 container init eb4af56c72691747ed85521c46111577d093ed98210ad1df6a56cbec99ecc8ea (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD) 2024-04-22T15:21:21.413 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 podman[54560]: 2024-04-22 15:21:21.308405707 +0000 UTC m=+0.442568139 container start eb4af56c72691747ed85521c46111577d093ed98210ad1df6a56cbec99ecc8ea (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux ) 2024-04-22T15:21:21.414 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 podman[54560]: 2024-04-22 15:21:21.4111048 +0000 UTC m=+0.545267244 container attach eb4af56c72691747ed85521c46111577d093ed98210ad1df6a56cbec99ecc8ea (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True) 2024-04-22T15:21:21.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 252852 10.64 MB/sec execute 450 sec latency 47.257 ms 2024-04-22T15:21:22.185 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate[54575]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:21:22.185 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate[54575]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:21:22.185 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 bash[54560]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:21:22.185 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 bash[54560]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:21:22.185 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 bash[54560]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T15:21:22.185 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 bash[54560]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T15:21:22.186 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 bash[54560]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T15:21:22.186 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 bash[54560]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T15:21:22.186 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 bash[54560]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-22T15:21:22.186 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 bash[54560]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T15:21:22.186 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 bash[54560]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-22T15:21:22.186 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate[54575]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T15:21:22.186 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate[54575]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T15:21:22.186 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate[54575]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T15:21:22.186 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate[54575]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T15:21:22.186 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate[54575]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-22T15:21:22.186 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate[54575]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T15:21:22.187 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate[54575]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-22T15:21:22.187 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:21 smithi186 podman[54560]: 2024-04-22 15:21:21.830556051 +0000 UTC m=+0.964718492 container died eb4af56c72691747ed85521c46111577d093ed98210ad1df6a56cbec99ecc8ea (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-22T15:21:22.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 253173 10.63 MB/sec execute 451 sec latency 64.314 ms 2024-04-22T15:21:22.736 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:22 smithi186 podman[54713]: 2024-04-22 15:21:22.48300411 +0000 UTC m=+0.647244369 container remove eb4af56c72691747ed85521c46111577d093ed98210ad1df6a56cbec99ecc8ea (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-22T15:21:22.736 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:22 smithi186 podman[54777]: 2024-04-22 15:21:22.601085358 +0000 UTC m=+0.019072926 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:21:22.736 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:22 smithi186 podman[54777]: 2024-04-22 15:21:22.733815024 +0000 UTC m=+0.151802590 container create 9518e2eb7665dff135d727a54ea6487618b412d6d483a88924e4e60c2c2d92b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-22T15:21:22.996 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:22 smithi186 podman[54777]: 2024-04-22 15:21:22.970342747 +0000 UTC m=+0.388330315 container init 9518e2eb7665dff135d727a54ea6487618b412d6d483a88924e4e60c2c2d92b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:21:22.996 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:22 smithi186 podman[54777]: 2024-04-22 15:21:22.973585428 +0000 UTC m=+0.391572996 container start 9518e2eb7665dff135d727a54ea6487618b412d6d483a88924e4e60c2c2d92b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:21:23.275 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:22 smithi186 ceph-osd[54796]: -- 172.21.15.186:0/2182673227 <== mon.1 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5632bdfd7680 con 0x5632bdfa1000 2024-04-22T15:21:23.275 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:23 smithi186 bash[54777]: 9518e2eb7665dff135d727a54ea6487618b412d6d483a88924e4e60c2c2d92b9 2024-04-22T15:21:23.275 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:23 smithi186 systemd[1]: Started Ceph osd.4 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:21:23.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 253478 10.62 MB/sec execute 452 sec latency 77.458 ms 2024-04-22T15:21:23.807 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:23 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:21:23.522+0000 7f03f937e7c0 -1 Falling back to public interface 2024-04-22T15:21:23.920 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:23 smithi119 ceph-mon[63156]: pgmap v200: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 5 peering, 56 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.2 MiB/s wr, 473 op/s; 246/1632 objects degraded (15.074%) 2024-04-22T15:21:23.920 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:23.920 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:23.921 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:24.189 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:23 smithi186 ceph-mon[49804]: pgmap v200: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 5 peering, 56 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.2 MiB/s wr, 473 op/s; 246/1632 objects degraded (15.074%) 2024-04-22T15:21:24.189 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:24.189 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:24.189 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:24.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 253727 10.61 MB/sec execute 453 sec latency 44.338 ms 2024-04-22T15:21:25.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 254022 10.61 MB/sec execute 454 sec latency 37.027 ms 2024-04-22T15:21:25.943 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:25 smithi119 ceph-mon[63156]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.5 MiB/s wr, 295 op/s; 258/1626 objects degraded (15.867%) 2024-04-22T15:21:25.943 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:25.943 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:25 smithi186 ceph-mon[49804]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.5 MiB/s wr, 295 op/s; 258/1626 objects degraded (15.867%) 2024-04-22T15:21:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:26.524 INFO:teuthology.orchestra.run.smithi119.stdout: 5 254349 10.61 MB/sec execute 455 sec latency 47.272 ms 2024-04-22T15:21:26.965 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:26 smithi119 ceph-mon[63156]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 453 op/s; 269/1656 objects degraded (16.244%) 2024-04-22T15:21:26.965 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:26 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 269/1656 objects degraded (16.244%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T15:21:27.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:26 smithi186 ceph-mon[49804]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 453 op/s; 269/1656 objects degraded (16.244%) 2024-04-22T15:21:27.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:26 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 269/1656 objects degraded (16.244%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T15:21:27.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 254733 10.61 MB/sec execute 456 sec latency 53.455 ms 2024-04-22T15:21:27.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:27 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:27.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:27 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:28.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:27 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:28.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:27 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:28.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 255041 10.61 MB/sec execute 457 sec latency 38.810 ms 2024-04-22T15:21:29.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 255299 10.59 MB/sec execute 458 sec latency 47.444 ms 2024-04-22T15:21:29.541 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:29 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:29.541 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:29 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:29.541 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:29 smithi119 ceph-mon[63156]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 407 op/s; 272/1686 objects degraded (16.133%) 2024-04-22T15:21:29.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:29 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:29.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:29 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:29.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:29 smithi186 ceph-mon[49804]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 407 op/s; 272/1686 objects degraded (16.133%) 2024-04-22T15:21:30.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 255676 10.59 MB/sec execute 459 sec latency 54.543 ms 2024-04-22T15:21:30.739 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:30 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:21:30.474+0000 7f03f937e7c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-22T15:21:31.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 255979 10.58 MB/sec execute 460 sec latency 54.658 ms 2024-04-22T15:21:31.825 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:31 smithi119 ceph-mon[63156]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 486 op/s; 264/1662 objects degraded (15.884%) 2024-04-22T15:21:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:31 smithi186 ceph-mon[49804]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 486 op/s; 264/1662 objects degraded (15.884%) 2024-04-22T15:21:32.025 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:31 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:21:31.896+0000 7f03f937e7c0 -1 osd.4 118 log_to_monitors true 2024-04-22T15:21:32.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 256263 10.58 MB/sec execute 461 sec latency 42.535 ms 2024-04-22T15:21:32.871 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='osd.4 [v2:172.21.15.186:6800/3579837926,v1:172.21.15.186:6801/3579837926]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:21:32.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:21:32.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:32.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:32.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:32.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:21:32.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:32.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:32.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:32.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:21:32.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:21:32.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:21:32.873 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:21:33.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='osd.4 [v2:172.21.15.186:6800/3579837926,v1:172.21.15.186:6801/3579837926]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:21:33.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:21:33.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:33.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:33.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:21:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:21:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:21:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:21:33.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:21:33.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 256472 10.57 MB/sec execute 462 sec latency 135.280 ms 2024-04-22T15:21:33.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:21:33 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:21:33.303+0000 7f03e67ce700 -1 osd.4 118 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:21:33.963 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:33 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:21:33.964 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:33 smithi119 ceph-mon[63156]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-22T15:21:33.964 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:33 smithi119 ceph-mon[63156]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.5 MiB/s wr, 330 op/s; 264/1662 objects degraded (15.884%) 2024-04-22T15:21:33.964 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:33 smithi119 ceph-mon[63156]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T15:21:33.964 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:33 smithi119 ceph-mon[63156]: from='osd.4 [v2:172.21.15.186:6800/3579837926,v1:172.21.15.186:6801/3579837926]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:21:33.964 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:33 smithi119 ceph-mon[63156]: osdmap e121: 8 total, 7 up, 8 in 2024-04-22T15:21:33.964 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:33 smithi119 ceph-mon[63156]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:21:33.964 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:33 smithi119 ceph-mon[63156]: from='osd.4 ' entity='osd.4' 2024-04-22T15:21:33.964 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:33 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 264/1662 objects degraded (15.884%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T15:21:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:33 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:21:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:33 smithi186 ceph-mon[49804]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-22T15:21:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:33 smithi186 ceph-mon[49804]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.5 MiB/s wr, 330 op/s; 264/1662 objects degraded (15.884%) 2024-04-22T15:21:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:33 smithi186 ceph-mon[49804]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T15:21:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:33 smithi186 ceph-mon[49804]: from='osd.4 [v2:172.21.15.186:6800/3579837926,v1:172.21.15.186:6801/3579837926]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:21:34.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:33 smithi186 ceph-mon[49804]: osdmap e121: 8 total, 7 up, 8 in 2024-04-22T15:21:34.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:33 smithi186 ceph-mon[49804]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:21:34.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:33 smithi186 ceph-mon[49804]: from='osd.4 ' entity='osd.4' 2024-04-22T15:21:34.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:33 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 264/1662 objects degraded (15.884%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T15:21:34.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 256789 10.57 MB/sec execute 463 sec latency 132.744 ms 2024-04-22T15:21:34.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:34 smithi119 ceph-mon[63156]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:21:34.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:34 smithi119 ceph-mon[63156]: osd.4 [v2:172.21.15.186:6800/3579837926,v1:172.21.15.186:6801/3579837926] boot 2024-04-22T15:21:34.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:34 smithi119 ceph-mon[63156]: osdmap e122: 8 total, 8 up, 8 in 2024-04-22T15:21:34.983 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:34 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:21:34.984 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:34 smithi119 ceph-mon[63156]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.5 MiB/s wr, 379 op/s; 262/1632 objects degraded (16.054%) 2024-04-22T15:21:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:34 smithi186 ceph-mon[49804]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:21:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:34 smithi186 ceph-mon[49804]: osd.4 [v2:172.21.15.186:6800/3579837926,v1:172.21.15.186:6801/3579837926] boot 2024-04-22T15:21:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:34 smithi186 ceph-mon[49804]: osdmap e122: 8 total, 8 up, 8 in 2024-04-22T15:21:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:34 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:21:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:34 smithi186 ceph-mon[49804]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.5 MiB/s wr, 379 op/s; 262/1632 objects degraded (16.054%) 2024-04-22T15:21:35.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 256934 10.55 MB/sec execute 464 sec latency 1026.467 ms 2024-04-22T15:21:36.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 257212 10.54 MB/sec execute 465 sec latency 851.985 ms 2024-04-22T15:21:36.531 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:36 smithi119 ceph-mon[63156]: osdmap e123: 8 total, 8 up, 8 in 2024-04-22T15:21:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:36 smithi186 ceph-mon[49804]: osdmap e123: 8 total, 8 up, 8 in 2024-04-22T15:21:37.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 257471 10.53 MB/sec execute 466 sec latency 77.526 ms 2024-04-22T15:21:37.559 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:37 smithi119 ceph-mon[63156]: osdmap e124: 8 total, 8 up, 8 in 2024-04-22T15:21:37.560 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:37 smithi119 ceph-mon[63156]: pgmap v211: 97 pgs: 2 active+recovery_wait+degraded, 4 remapped+peering, 12 active+undersized, 14 active+undersized+degraded, 65 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 632 op/s; 168/1632 objects degraded (10.294%); 0 B/s, 0 keys/s, 12 objects/s recovering 2024-04-22T15:21:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:37 smithi186 ceph-mon[49804]: osdmap e124: 8 total, 8 up, 8 in 2024-04-22T15:21:37.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:37 smithi186 ceph-mon[49804]: pgmap v211: 97 pgs: 2 active+recovery_wait+degraded, 4 remapped+peering, 12 active+undersized, 14 active+undersized+degraded, 65 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 632 op/s; 168/1632 objects degraded (10.294%); 0 B/s, 0 keys/s, 12 objects/s recovering 2024-04-22T15:21:38.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 257746 10.53 MB/sec execute 467 sec latency 146.919 ms 2024-04-22T15:21:38.580 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:38 smithi119 ceph-mon[63156]: osdmap e125: 8 total, 8 up, 8 in 2024-04-22T15:21:38.676 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:38 smithi186 ceph-mon[49804]: osdmap e125: 8 total, 8 up, 8 in 2024-04-22T15:21:39.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 257902 10.51 MB/sec execute 468 sec latency 1018.038 ms 2024-04-22T15:21:39.645 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:39 smithi119 ceph-mon[63156]: osdmap e126: 8 total, 8 up, 8 in 2024-04-22T15:21:39.645 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:39 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 168/1632 objects degraded (10.294%), 16 pgs degraded (PG_DEGRADED) 2024-04-22T15:21:39.645 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:39 smithi119 ceph-mon[63156]: pgmap v214: 97 pgs: 2 active+recovery_wait+degraded, 4 remapped+peering, 2 active+undersized, 8 active+undersized+degraded, 81 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 2.9 MiB/s wr, 392 op/s; 108/1638 objects degraded (6.593%); 397 KiB/s, 0 keys/s, 28 objects/s recovering 2024-04-22T15:21:39.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:39 smithi186 ceph-mon[49804]: osdmap e126: 8 total, 8 up, 8 in 2024-04-22T15:21:39.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:39 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 168/1632 objects degraded (10.294%), 16 pgs degraded (PG_DEGRADED) 2024-04-22T15:21:39.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:39 smithi186 ceph-mon[49804]: pgmap v214: 97 pgs: 2 active+recovery_wait+degraded, 4 remapped+peering, 2 active+undersized, 8 active+undersized+degraded, 81 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 2.9 MiB/s wr, 392 op/s; 108/1638 objects degraded (6.593%); 397 KiB/s, 0 keys/s, 28 objects/s recovering 2024-04-22T15:21:40.452 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:40 smithi119 ceph-mon[63156]: osdmap e127: 8 total, 8 up, 8 in 2024-04-22T15:21:40.452 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:40 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:40.453 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:40 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:40.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 258126 10.51 MB/sec execute 469 sec latency 536.287 ms 2024-04-22T15:21:40.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:40 smithi186 ceph-mon[49804]: osdmap e127: 8 total, 8 up, 8 in 2024-04-22T15:21:40.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:40 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:40.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:40 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:41.451 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:41 smithi119 ceph-mon[63156]: osdmap e128: 8 total, 8 up, 8 in 2024-04-22T15:21:41.451 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:41 smithi119 ceph-mon[63156]: pgmap v217: 97 pgs: 1 peering, 96 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 441 op/s; 1.7 MiB/s, 59 keys/s, 63 objects/s recovering 2024-04-22T15:21:41.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 258602 10.52 MB/sec execute 470 sec latency 36.750 ms 2024-04-22T15:21:41.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:41 smithi186 ceph-mon[49804]: osdmap e128: 8 total, 8 up, 8 in 2024-04-22T15:21:41.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:41 smithi186 ceph-mon[49804]: pgmap v217: 97 pgs: 1 peering, 96 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 441 op/s; 1.7 MiB/s, 59 keys/s, 63 objects/s recovering 2024-04-22T15:21:42.466 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:42 smithi119 ceph-mon[63156]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 108/1638 objects degraded (6.593%), 10 pgs degraded) 2024-04-22T15:21:42.510 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:42 smithi119 ceph-mon[63156]: Cluster is now healthy 2024-04-22T15:21:42.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 258980 10.51 MB/sec execute 471 sec latency 33.010 ms 2024-04-22T15:21:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:42 smithi186 ceph-mon[49804]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 108/1638 objects degraded (6.593%), 10 pgs degraded) 2024-04-22T15:21:42.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:42 smithi186 ceph-mon[49804]: Cluster is now healthy 2024-04-22T15:21:43.163 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:21:43.480 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:43 smithi119 ceph-mon[63156]: pgmap v218: 97 pgs: 1 peering, 96 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.3 MiB/s wr, 326 op/s; 1.2 MiB/s, 44 keys/s, 46 objects/s recovering 2024-04-22T15:21:43.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 259317 10.51 MB/sec execute 472 sec latency 43.912 ms 2024-04-22T15:21:43.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:43 smithi186 ceph-mon[49804]: pgmap v218: 97 pgs: 1 peering, 96 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.3 MiB/s wr, 326 op/s; 1.2 MiB/s, 44 keys/s, 46 objects/s recovering 2024-04-22T15:21:44.161 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:21:44.161 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (7m) 56s ago 16m 15.0M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:21:44.161 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (4m) 56s ago 16m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:21:44.161 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (3m) 17s ago 15m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:21:44.161 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (7m) 56s ago 16m 47.8M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (10m) 56s ago 10m 38.4M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (10m) 17s ago 10m 14.9M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (6m) 56s ago 17m 509M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (5m) 17s ago 15m 452M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (5m) 56s ago 17m 52.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (5m) 17s ago 15m 43.2M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (10m) 56s ago 10m 285M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (10m) 17s ago 9m 84.8M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (7m) 56s ago 16m 10.4M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (6m) 17s ago 15m 18.2M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (3m) 56s ago 14m 357M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (2m) 56s ago 14m 286M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:21:44.162 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (101s) 56s ago 13m 220M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:21:44.163 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (66s) 56s ago 13m 12.8M 1936M 19.0.0-3223-g5883d494 3d983c26e278 2378efa46eca 2024-04-22T15:21:44.163 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (21s) 17s ago 13m 14.8M 2704M 19.0.0-3223-g5883d494 3d983c26e278 9518e2eb7665 2024-04-22T15:21:44.163 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (12m) 17s ago 12m 596M 2704M 17.2.0 e1d6a67b021e 438254d88b68 2024-04-22T15:21:44.163 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (12m) 17s ago 12m 522M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:21:44.163 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (11m) 17s ago 11m 567M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:21:44.163 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (5m) 56s ago 15m 68.0M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:21:44.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:44 smithi119 ceph-mon[63156]: from='client.44287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:44.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:44 smithi119 ceph-mon[63156]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:44.525 INFO:teuthology.orchestra.run.smithi119.stdout: 5 259695 10.51 MB/sec execute 473 sec latency 43.836 ms 2024-04-22T15:21:44.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:44 smithi186 ceph-mon[49804]: from='client.44287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:44.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:44 smithi186 ceph-mon[49804]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:44.789 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:21:44.789 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:21:44.789 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:21:44.789 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:21:44.789 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:21:44.789 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:21:44.789 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:21:44.789 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:21:44.789 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-22T15:21:44.789 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 5 2024-04-22T15:21:44.789 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:21:44.790 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:21:44.790 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:21:44.790 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:21:44.790 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:21:44.790 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-22T15:21:44.790 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 9 2024-04-22T15:21:44.790 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:21:44.790 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:21:45.250 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:21:45.250 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:21:45.250 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:21:45.250 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:21:45.250 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:21:45.250 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:21:45.250 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:21:45.250 INFO:teuthology.orchestra.run.smithi119.stdout: "crash" 2024-04-22T15:21:45.250 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:21:45.250 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "11/23 daemons upgraded", 2024-04-22T15:21:45.250 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:21:45.251 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:21:45.251 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:21:45.500 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:45 smithi119 ceph-mon[63156]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:45.500 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:45 smithi119 ceph-mon[63156]: pgmap v219: 97 pgs: 97 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.9 MiB/s wr, 483 op/s; 827 KiB/s, 37 keys/s, 29 objects/s recovering 2024-04-22T15:21:45.500 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:45 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/370896437' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:21:45.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 260028 10.50 MB/sec execute 474 sec latency 72.820 ms 2024-04-22T15:21:45.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:45 smithi186 ceph-mon[49804]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:45.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:45 smithi186 ceph-mon[49804]: pgmap v219: 97 pgs: 97 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.9 MiB/s wr, 483 op/s; 827 KiB/s, 37 keys/s, 29 objects/s recovering 2024-04-22T15:21:45.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:45 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/370896437' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:21:45.979 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:21:46.525 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:46 smithi119 ceph-mon[63156]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:46.526 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:46 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/2424902280' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:21:46.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 260387 10.50 MB/sec execute 475 sec latency 96.236 ms 2024-04-22T15:21:46.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:46 smithi186 ceph-mon[49804]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:46.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:46 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/2424902280' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:21:47.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 260772 10.50 MB/sec execute 476 sec latency 57.574 ms 2024-04-22T15:21:47.553 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:47 smithi119 ceph-mon[63156]: pgmap v220: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 431 op/s; 662 KiB/s, 29 keys/s, 23 objects/s recovering 2024-04-22T15:21:47.553 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:47 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:21:47.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:47 smithi186 ceph-mon[49804]: pgmap v220: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 431 op/s; 662 KiB/s, 29 keys/s, 23 objects/s recovering 2024-04-22T15:21:47.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:47 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:21:48.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 261060 10.49 MB/sec execute 477 sec latency 50.699 ms 2024-04-22T15:21:48.589 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:48 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:21:48.589 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:48 smithi119 ceph-mon[63156]: Upgrade: osd.5 is safe to restart 2024-04-22T15:21:48.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:48 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:21:48.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:48 smithi186 ceph-mon[49804]: Upgrade: osd.5 is safe to restart 2024-04-22T15:21:49.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 261413 10.49 MB/sec execute 478 sec latency 44.183 ms 2024-04-22T15:21:49.611 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:49 smithi119 ceph-mon[63156]: pgmap v221: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 494 op/s; 436 KiB/s, 10 objects/s recovering 2024-04-22T15:21:49.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:49 smithi186 ceph-mon[49804]: pgmap v221: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 494 op/s; 436 KiB/s, 10 objects/s recovering 2024-04-22T15:21:50.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 261607 10.48 MB/sec execute 479 sec latency 63.278 ms 2024-04-22T15:21:50.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:50 smithi119 ceph-mon[63156]: Upgrade: Updating osd.5 2024-04-22T15:21:50.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:50 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:50.639 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:50 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:21:50.640 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:50 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:50.640 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:50 smithi119 ceph-mon[63156]: Deploying daemon osd.5 on smithi186 2024-04-22T15:21:50.640 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:50 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:50.701 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:50 smithi186 ceph-mon[49804]: Upgrade: Updating osd.5 2024-04-22T15:21:50.702 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:50 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:50.702 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:50 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:21:50.702 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:50 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:50.702 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:50 smithi186 ceph-mon[49804]: Deploying daemon osd.5 on smithi186 2024-04-22T15:21:50.702 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:50 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:51.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 261858 10.47 MB/sec execute 480 sec latency 62.381 ms 2024-04-22T15:21:51.681 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:51 smithi119 ceph-mon[63156]: pgmap v222: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.8 MiB/s wr, 629 op/s; 394 KiB/s, 9 objects/s recovering 2024-04-22T15:21:51.700 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:51 smithi186 ceph-mon[49804]: pgmap v222: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.8 MiB/s wr, 629 op/s; 394 KiB/s, 9 objects/s recovering 2024-04-22T15:21:52.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 262155 10.48 MB/sec execute 481 sec latency 68.789 ms 2024-04-22T15:21:53.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 262350 10.47 MB/sec execute 482 sec latency 84.640 ms 2024-04-22T15:21:53.760 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:53 smithi186 systemd[1]: Stopping Ceph osd.5 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:21:53.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:53 smithi119 ceph-mon[63156]: pgmap v223: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 429 op/s 2024-04-22T15:21:53.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:53 smithi119 ceph-mon[63156]: osd.5 marked itself down and dead 2024-04-22T15:21:54.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:53 smithi186 ceph-mon[49804]: pgmap v223: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 429 op/s 2024-04-22T15:21:54.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:53 smithi186 ceph-mon[49804]: osd.5 marked itself down and dead 2024-04-22T15:21:54.025 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:53 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[31169]: 2024-04-22T15:21:53.791+0000 7fb96bc9b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:21:54.025 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:53 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[31169]: 2024-04-22T15:21:53.791+0000 7fb96bc9b700 -1 osd.5 128 *** Got signal Terminated *** 2024-04-22T15:21:54.025 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:53 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[31169]: 2024-04-22T15:21:53.791+0000 7fb96bc9b700 -1 osd.5 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:21:54.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 262418 10.45 MB/sec execute 483 sec latency 718.760 ms 2024-04-22T15:21:54.758 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:54 smithi186 podman[56584]: 2024-04-22 15:21:54.484307016 +0000 UTC m=+0.828691286 container died 438254d88b681eef359b2fbaee9ecd072464e2cc348b2f991fd0ae47ff706172 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-type=git, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, architecture=x86_64, version=8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_CLEAN=True, io.buildah.version=1.19.8, name=centos-stream) 2024-04-22T15:21:55.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:54 smithi186 ceph-mon[49804]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:21:55.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:54 smithi186 ceph-mon[49804]: osdmap e129: 8 total, 7 up, 8 in 2024-04-22T15:21:55.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:54 smithi186 ceph-mon[49804]: pgmap v225: 97 pgs: 7 peering, 11 stale+active+clean, 79 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 474 op/s 2024-04-22T15:21:55.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:54 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:55.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:54 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:55.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:54 smithi186 ceph-mon[49804]: osdmap e130: 8 total, 7 up, 8 in 2024-04-22T15:21:55.026 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:54 smithi186 podman[56584]: 2024-04-22 15:21:54.947126372 +0000 UTC m=+1.291510688 container cleanup 438254d88b681eef359b2fbaee9ecd072464e2cc348b2f991fd0ae47ff706172 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, RELEASE=HEAD, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, release=754, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-22T15:21:55.026 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:54 smithi186 bash[56584]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5 2024-04-22T15:21:55.097 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:54 smithi119 ceph-mon[63156]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:21:55.097 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:54 smithi119 ceph-mon[63156]: osdmap e129: 8 total, 7 up, 8 in 2024-04-22T15:21:55.097 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:54 smithi119 ceph-mon[63156]: pgmap v225: 97 pgs: 7 peering, 11 stale+active+clean, 79 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 474 op/s 2024-04-22T15:21:55.098 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:54 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:21:55.098 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:54 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:55.098 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:54 smithi119 ceph-mon[63156]: osdmap e130: 8 total, 7 up, 8 in 2024-04-22T15:21:55.525 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:55 smithi186 podman[56600]: 2024-04-22 15:21:55.265529717 +0000 UTC m=+0.771071634 container remove 438254d88b681eef359b2fbaee9ecd072464e2cc348b2f991fd0ae47ff706172 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5, RELEASE=HEAD, release=754, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., vcs-type=git, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, version=8, distribution-scope=public, GIT_BRANCH=HEAD, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-22T15:21:55.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 262654 10.44 MB/sec execute 484 sec latency 1077.242 ms 2024-04-22T15:21:55.967 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:55 smithi186 podman[56693]: 2024-04-22 15:21:55.574021815 +0000 UTC m=+0.020142659 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:21:55.967 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:55 smithi186 podman[56693]: 2024-04-22 15:21:55.841693038 +0000 UTC m=+0.287813863 container create 22a956c9c551f38af031ed1b2f9724bc39cbc8a21ace22796fd22251cbdb64b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:21:56.275 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:56 smithi186 podman[56693]: 2024-04-22 15:21:56.103608679 +0000 UTC m=+0.549729510 container init 22a956c9c551f38af031ed1b2f9724bc39cbc8a21ace22796fd22251cbdb64b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416) 2024-04-22T15:21:56.275 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:56 smithi186 podman[56693]: 2024-04-22 15:21:56.107130794 +0000 UTC m=+0.553251627 container start 22a956c9c551f38af031ed1b2f9724bc39cbc8a21ace22796fd22251cbdb64b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-22T15:21:56.275 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:56 smithi186 podman[56693]: 2024-04-22 15:21:56.194918049 +0000 UTC m=+0.641038882 container attach 22a956c9c551f38af031ed1b2f9724bc39cbc8a21ace22796fd22251cbdb64b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS) 2024-04-22T15:21:56.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 263028 10.43 MB/sec execute 485 sec latency 59.532 ms 2024-04-22T15:21:56.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:56 smithi186 podman[56693]: 2024-04-22 15:21:56.468941568 +0000 UTC m=+0.915062405 container died 22a956c9c551f38af031ed1b2f9724bc39cbc8a21ace22796fd22251cbdb64b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-22T15:21:56.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:56 smithi119 ceph-mon[63156]: pgmap v227: 97 pgs: 4 active+undersized, 10 active+undersized+degraded, 7 peering, 6 stale+active+clean, 70 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.2 MiB/s wr, 414 op/s; 68/1656 objects degraded (4.106%) 2024-04-22T15:21:56.872 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:56 smithi119 ceph-mon[63156]: Health check failed: Degraded data redundancy: 68/1656 objects degraded (4.106%), 10 pgs degraded (PG_DEGRADED) 2024-04-22T15:21:57.113 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:56 smithi186 ceph-mon[49804]: pgmap v227: 97 pgs: 4 active+undersized, 10 active+undersized+degraded, 7 peering, 6 stale+active+clean, 70 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.2 MiB/s wr, 414 op/s; 68/1656 objects degraded (4.106%) 2024-04-22T15:21:57.114 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:56 smithi186 ceph-mon[49804]: Health check failed: Degraded data redundancy: 68/1656 objects degraded (4.106%), 10 pgs degraded (PG_DEGRADED) 2024-04-22T15:21:57.114 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:57 smithi186 podman[56719]: 2024-04-22 15:21:57.07499355 +0000 UTC m=+0.600841273 container remove 22a956c9c551f38af031ed1b2f9724bc39cbc8a21ace22796fd22251cbdb64b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux ) 2024-04-22T15:21:57.114 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:57 smithi186 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.5.service: Deactivated successfully. 2024-04-22T15:21:57.114 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:57 smithi186 systemd[1]: Stopped Ceph osd.5 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:21:57.114 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:57 smithi186 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.5.service: Consumed 5min 32.392s CPU time. 2024-04-22T15:21:57.525 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:57 smithi186 systemd[1]: Starting Ceph osd.5 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:21:57.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 263400 10.43 MB/sec execute 486 sec latency 62.354 ms 2024-04-22T15:21:57.840 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:57 smithi186 podman[56824]: 2024-04-22 15:21:57.464516136 +0000 UTC m=+0.020402354 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:21:57.840 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:57 smithi186 podman[56824]: 2024-04-22 15:21:57.584552833 +0000 UTC m=+0.140439033 container create 40011b8dfc9a8ebde59823114df543eb51b6205374e526961d33844f08274f82 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-04-22T15:21:57.840 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:57 smithi186 podman[56824]: 2024-04-22 15:21:57.837630839 +0000 UTC m=+0.393517043 container init 40011b8dfc9a8ebde59823114df543eb51b6205374e526961d33844f08274f82 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=) 2024-04-22T15:21:58.275 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:57 smithi186 podman[56824]: 2024-04-22 15:21:57.841409 +0000 UTC m=+0.397295214 container start 40011b8dfc9a8ebde59823114df543eb51b6205374e526961d33844f08274f82 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-22T15:21:58.275 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:57 smithi186 podman[56824]: 2024-04-22 15:21:57.937833218 +0000 UTC m=+0.493719429 container attach 40011b8dfc9a8ebde59823114df543eb51b6205374e526961d33844f08274f82 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240416) 2024-04-22T15:21:58.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 263746 10.43 MB/sec execute 487 sec latency 49.231 ms 2024-04-22T15:21:58.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate[56839]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:21:58.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate[56839]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:21:58.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 bash[56824]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:21:58.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 bash[56824]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:21:58.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 bash[56824]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T15:21:58.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 bash[56824]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T15:21:58.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 bash[56824]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T15:21:58.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 bash[56824]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T15:21:58.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 bash[56824]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-22T15:21:58.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 bash[56824]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T15:21:58.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 bash[56824]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-22T15:21:58.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate[56839]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T15:21:58.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate[56839]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T15:21:58.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate[56839]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T15:21:58.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate[56839]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T15:21:58.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate[56839]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-22T15:21:58.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate[56839]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T15:21:58.777 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate[56839]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-22T15:21:58.777 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:58 smithi186 podman[56824]: 2024-04-22 15:21:58.453816553 +0000 UTC m=+1.009702759 container died 40011b8dfc9a8ebde59823114df543eb51b6205374e526961d33844f08274f82 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:21:59.373 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:59 smithi186 podman[56976]: 2024-04-22 15:21:59.120110104 +0000 UTC m=+0.660899556 container remove 40011b8dfc9a8ebde59823114df543eb51b6205374e526961d33844f08274f82 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:21:59.373 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:59 smithi186 podman[57068]: 2024-04-22 15:21:59.251864938 +0000 UTC m=+0.022239202 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:21:59.374 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:59 smithi186 podman[57068]: 2024-04-22 15:21:59.371229117 +0000 UTC m=+0.141603363 container create e1c8f1367a89bcf8496f086a8ba91bcbb76a0341d0017d57a533ab019694069c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240416, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS) 2024-04-22T15:21:59.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 264147 10.43 MB/sec execute 488 sec latency 44.213 ms 2024-04-22T15:21:59.626 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:21:59 smithi186 ceph-mon[49804]: pgmap v228: 97 pgs: 6 active+undersized, 18 active+undersized+degraded, 7 peering, 66 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.9 MiB/s wr, 214 op/s; 133/1698 objects degraded (7.833%) 2024-04-22T15:21:59.686 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:21:59 smithi119 ceph-mon[63156]: pgmap v228: 97 pgs: 6 active+undersized, 18 active+undersized+degraded, 7 peering, 66 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.9 MiB/s wr, 214 op/s; 133/1698 objects degraded (7.833%) 2024-04-22T15:22:00.025 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:59 smithi186 podman[57068]: 2024-04-22 15:21:59.624242602 +0000 UTC m=+0.394616852 container init e1c8f1367a89bcf8496f086a8ba91bcbb76a0341d0017d57a533ab019694069c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:22:00.025 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:59 smithi186 podman[57068]: 2024-04-22 15:21:59.62867184 +0000 UTC m=+0.399046078 container start e1c8f1367a89bcf8496f086a8ba91bcbb76a0341d0017d57a533ab019694069c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-22T15:22:00.025 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:59 smithi186 ceph-osd[57087]: -- 172.21.15.186:0/1417380134 <== mon.1 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f9d79bd680 con 0x55f9d7986c00 2024-04-22T15:22:00.025 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:59 smithi186 bash[57068]: e1c8f1367a89bcf8496f086a8ba91bcbb76a0341d0017d57a533ab019694069c 2024-04-22T15:22:00.025 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:21:59 smithi186 systemd[1]: Started Ceph osd.5 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:22:00.525 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:22:00 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:22:00.169+0000 7f01942327c0 -1 Falling back to public interface 2024-04-22T15:22:00.526 INFO:teuthology.orchestra.run.smithi119.stdout: 5 264507 10.43 MB/sec execute 489 sec latency 68.127 ms 2024-04-22T15:22:00.966 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:00 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:00.967 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:00 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:00.967 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:00 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:22:01.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:00 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:01.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:00 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:01.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:00 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:22:01.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 264868 10.42 MB/sec execute 490 sec latency 74.263 ms 2024-04-22T15:22:01.989 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:01 smithi119 ceph-mon[63156]: pgmap v229: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 521 op/s; 170/1716 objects degraded (9.907%) 2024-04-22T15:22:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:01 smithi186 ceph-mon[49804]: pgmap v229: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 521 op/s; 170/1716 objects degraded (9.907%) 2024-04-22T15:22:02.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 265328 10.43 MB/sec execute 491 sec latency 36.015 ms 2024-04-22T15:22:03.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 265781 10.44 MB/sec execute 492 sec latency 38.744 ms 2024-04-22T15:22:04.016 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:03 smithi119 ceph-mon[63156]: pgmap v230: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.1 MiB/s wr, 442 op/s; 170/1716 objects degraded (9.907%) 2024-04-22T15:22:04.017 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:03 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 170/1716 objects degraded (9.907%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:04.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:03 smithi186 ceph-mon[49804]: pgmap v230: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.1 MiB/s wr, 442 op/s; 170/1716 objects degraded (9.907%) 2024-04-22T15:22:04.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:03 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 170/1716 objects degraded (9.907%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:04.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 266128 10.44 MB/sec execute 493 sec latency 38.942 ms 2024-04-22T15:22:05.036 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:04 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:05.036 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:04 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:05.037 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:04 smithi119 ceph-mon[63156]: pgmap v231: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 439 op/s; 181/1731 objects degraded (10.456%) 2024-04-22T15:22:05.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:04 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:05.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:04 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:05.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:04 smithi186 ceph-mon[49804]: pgmap v231: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 439 op/s; 181/1731 objects degraded (10.456%) 2024-04-22T15:22:05.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 266406 10.43 MB/sec execute 494 sec latency 52.911 ms 2024-04-22T15:22:06.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:06.311 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:06.490 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:06.490 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:06.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 266700 10.42 MB/sec execute 495 sec latency 61.611 ms 2024-04-22T15:22:07.332 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:07 smithi119 ceph-mon[63156]: pgmap v232: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.5 MiB/s wr, 415 op/s; 184/1719 objects degraded (10.704%) 2024-04-22T15:22:07.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:07 smithi186 ceph-mon[49804]: pgmap v232: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.5 MiB/s wr, 415 op/s; 184/1719 objects degraded (10.704%) 2024-04-22T15:22:07.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 266998 10.41 MB/sec execute 496 sec latency 58.227 ms 2024-04-22T15:22:08.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 267260 10.41 MB/sec execute 497 sec latency 69.636 ms 2024-04-22T15:22:08.617 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:08 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 184/1719 objects degraded (10.704%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:08.863 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:22:08 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:22:08.711+0000 7f01942327c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-22T15:22:08.863 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:08 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 184/1719 objects degraded (10.704%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:09.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 267515 10.41 MB/sec execute 498 sec latency 72.518 ms 2024-04-22T15:22:09.650 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:09 smithi119 ceph-mon[63156]: pgmap v233: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 453 op/s; 184/1692 objects degraded (10.875%) 2024-04-22T15:22:09.651 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:09.651 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:09.651 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:22:09.651 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:22:09.651 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:09.651 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:09.651 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:22:09.651 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:09.651 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:09.651 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:09.651 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:22:09.892 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:09 smithi186 ceph-mon[49804]: pgmap v233: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 453 op/s; 184/1692 objects degraded (10.875%) 2024-04-22T15:22:09.892 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:09.892 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:09.892 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:22:09.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:22:09.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:09.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:09.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:22:09.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:09.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:09.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:09.893 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:22:10.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 267842 10.40 MB/sec execute 499 sec latency 71.505 ms 2024-04-22T15:22:10.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:22:10 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:22:10.273+0000 7f01942327c0 -1 osd.5 128 log_to_monitors true 2024-04-22T15:22:10.938 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:10 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:22:10.938 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:10 smithi119 ceph-mon[63156]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-22T15:22:10.939 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:10.939 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:10.939 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:10 smithi119 ceph-mon[63156]: from='osd.5 [v2:172.21.15.186:6808/623366049,v1:172.21.15.186:6809/623366049]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:22:10.939 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:10 smithi119 ceph-mon[63156]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:22:11.275 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:22:10 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:22:10.829+0000 7f018568a700 -1 osd.5 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:22:11.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:10 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:22:11.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:10 smithi186 ceph-mon[49804]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-22T15:22:11.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:11.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:11.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:10 smithi186 ceph-mon[49804]: from='osd.5 [v2:172.21.15.186:6808/623366049,v1:172.21.15.186:6809/623366049]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:22:11.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:10 smithi186 ceph-mon[49804]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:22:11.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 268171 10.40 MB/sec execute 500 sec latency 43.855 ms 2024-04-22T15:22:11.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:11 smithi119 ceph-mon[63156]: pgmap v234: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.8 MiB/s wr, 627 op/s; 187/1632 objects degraded (11.458%) 2024-04-22T15:22:11.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:11 smithi119 ceph-mon[63156]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T15:22:11.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:11 smithi119 ceph-mon[63156]: osdmap e131: 8 total, 7 up, 8 in 2024-04-22T15:22:11.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:11 smithi119 ceph-mon[63156]: from='osd.5 [v2:172.21.15.186:6808/623366049,v1:172.21.15.186:6809/623366049]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:22:11.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:11 smithi119 ceph-mon[63156]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:22:12.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:11 smithi186 ceph-mon[49804]: pgmap v234: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.8 MiB/s wr, 627 op/s; 187/1632 objects degraded (11.458%) 2024-04-22T15:22:12.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:11 smithi186 ceph-mon[49804]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T15:22:12.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:11 smithi186 ceph-mon[49804]: osdmap e131: 8 total, 7 up, 8 in 2024-04-22T15:22:12.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:11 smithi186 ceph-mon[49804]: from='osd.5 [v2:172.21.15.186:6808/623366049,v1:172.21.15.186:6809/623366049]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:22:12.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:11 smithi186 ceph-mon[49804]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:22:12.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 268305 10.38 MB/sec execute 501 sec latency 699.667 ms 2024-04-22T15:22:13.064 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:12 smithi119 ceph-mon[63156]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:22:13.064 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:12 smithi119 ceph-mon[63156]: osd.5 [v2:172.21.15.186:6808/623366049,v1:172.21.15.186:6809/623366049] boot 2024-04-22T15:22:13.064 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:12 smithi119 ceph-mon[63156]: osdmap e132: 8 total, 8 up, 8 in 2024-04-22T15:22:13.065 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:12 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:22:13.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:12 smithi186 ceph-mon[49804]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:22:13.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:12 smithi186 ceph-mon[49804]: osd.5 [v2:172.21.15.186:6808/623366049,v1:172.21.15.186:6809/623366049] boot 2024-04-22T15:22:13.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:12 smithi186 ceph-mon[49804]: osdmap e132: 8 total, 8 up, 8 in 2024-04-22T15:22:13.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:12 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:22:13.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 268460 10.37 MB/sec execute 502 sec latency 1662.245 ms 2024-04-22T15:22:14.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:13 smithi119 ceph-mon[63156]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 348 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.1 MiB/s wr, 466 op/s; 187/1632 objects degraded (11.458%) 2024-04-22T15:22:14.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:13 smithi119 ceph-mon[63156]: osdmap e133: 8 total, 8 up, 8 in 2024-04-22T15:22:14.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:13 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 187/1632 objects degraded (11.458%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:14.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:13 smithi119 ceph-mon[63156]: osdmap e134: 8 total, 8 up, 8 in 2024-04-22T15:22:14.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:13 smithi186 ceph-mon[49804]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 348 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.1 MiB/s wr, 466 op/s; 187/1632 objects degraded (11.458%) 2024-04-22T15:22:14.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:13 smithi186 ceph-mon[49804]: osdmap e133: 8 total, 8 up, 8 in 2024-04-22T15:22:14.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:13 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 187/1632 objects degraded (11.458%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:14.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:13 smithi186 ceph-mon[49804]: osdmap e134: 8 total, 8 up, 8 in 2024-04-22T15:22:14.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 268508 10.35 MB/sec execute 503 sec latency 2425.538 ms 2024-04-22T15:22:15.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 268895 10.36 MB/sec execute 504 sec latency 37.133 ms 2024-04-22T15:22:16.060 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:15 smithi119 ceph-mon[63156]: pgmap v240: 97 pgs: 4 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+undersized, 14 active+undersized+degraded, 73 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s wr, 155 op/s; 169/1629 objects degraded (10.374%); 104 KiB/s, 5 objects/s recovering 2024-04-22T15:22:16.060 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:15 smithi119 ceph-mon[63156]: osdmap e135: 8 total, 8 up, 8 in 2024-04-22T15:22:16.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:15 smithi186 ceph-mon[49804]: pgmap v240: 97 pgs: 4 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+undersized, 14 active+undersized+degraded, 73 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s wr, 155 op/s; 169/1629 objects degraded (10.374%); 104 KiB/s, 5 objects/s recovering 2024-04-22T15:22:16.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:15 smithi186 ceph-mon[49804]: osdmap e135: 8 total, 8 up, 8 in 2024-04-22T15:22:16.498 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:22:16.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 269167 10.35 MB/sec execute 505 sec latency 547.672 ms 2024-04-22T15:22:17.085 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:16 smithi119 ceph-mon[63156]: osdmap e136: 8 total, 8 up, 8 in 2024-04-22T15:22:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:16 smithi186 ceph-mon[49804]: osdmap e136: 8 total, 8 up, 8 in 2024-04-22T15:22:17.449 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:22:17.449 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (7m) 89s ago 16m 15.0M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:22:17.449 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (4m) 89s ago 16m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:22:17.449 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (4m) 13s ago 16m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:22:17.449 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (7m) 89s ago 16m 47.8M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:22:17.449 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (10m) 89s ago 10m 38.4M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:22:17.449 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (10m) 13s ago 10m 15.0M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (7m) 89s ago 18m 509M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (6m) 13s ago 16m 452M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (6m) 89s ago 18m 52.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (5m) 13s ago 16m 41.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (10m) 89s ago 10m 285M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (10m) 13s ago 10m 84.9M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (7m) 89s ago 16m 10.4M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (7m) 13s ago 16m 17.9M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (3m) 89s ago 15m 357M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (3m) 89s ago 14m 286M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (2m) 89s ago 14m 220M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (99s) 89s ago 14m 12.8M 1936M 19.0.0-3223-g5883d494 3d983c26e278 2378efa46eca 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (54s) 13s ago 13m 275M 2704M 19.0.0-3223-g5883d494 3d983c26e278 9518e2eb7665 2024-04-22T15:22:17.450 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (18s) 13s ago 13m 14.7M 2704M 19.0.0-3223-g5883d494 3d983c26e278 e1c8f1367a89 2024-04-22T15:22:17.451 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (12m) 13s ago 12m 555M 2704M 17.2.0 e1d6a67b021e 2357de993a90 2024-04-22T15:22:17.451 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (12m) 13s ago 12m 601M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:22:17.451 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (6m) 89s ago 15m 68.0M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:22:17.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 269444 10.35 MB/sec execute 506 sec latency 883.699 ms 2024-04-22T15:22:17.836 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:17 smithi119 ceph-mon[63156]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:17.836 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:17 smithi119 ceph-mon[63156]: pgmap v243: 97 pgs: 3 peering, 3 activating+undersized+degraded+remapped, 2 remapped+peering, 1 active+undersized+remapped, 88 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.1 MiB/s wr, 279 op/s; 64/1602 objects degraded (3.995%); 1.2 MiB/s, 30 objects/s recovering 2024-04-22T15:22:17.836 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:17 smithi119 ceph-mon[63156]: osdmap e137: 8 total, 8 up, 8 in 2024-04-22T15:22:18.035 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:22:18.036 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:22:18.036 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:22:18.036 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:22:18.036 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:22:18.036 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:22:18.036 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:22:18.036 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:22:18.036 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-22T15:22:18.036 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 6 2024-04-22T15:22:18.036 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:22:18.036 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:22:18.036 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:22:18.037 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:22:18.037 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:22:18.037 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-22T15:22:18.037 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 10 2024-04-22T15:22:18.037 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:22:18.037 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:22:18.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:17 smithi186 ceph-mon[49804]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:18.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:17 smithi186 ceph-mon[49804]: pgmap v243: 97 pgs: 3 peering, 3 activating+undersized+degraded+remapped, 2 remapped+peering, 1 active+undersized+remapped, 88 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.1 MiB/s wr, 279 op/s; 64/1602 objects degraded (3.995%); 1.2 MiB/s, 30 objects/s recovering 2024-04-22T15:22:18.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:17 smithi186 ceph-mon[49804]: osdmap e137: 8 total, 8 up, 8 in 2024-04-22T15:22:18.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 269746 10.35 MB/sec execute 507 sec latency 48.743 ms 2024-04-22T15:22:18.542 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:22:18.542 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:22:18.542 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:22:18.542 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:22:18.543 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:22:18.543 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:22:18.543 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:22:18.543 INFO:teuthology.orchestra.run.smithi119.stdout: "crash" 2024-04-22T15:22:18.543 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:22:18.543 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "12/23 daemons upgraded", 2024-04-22T15:22:18.543 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:22:18.543 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:22:18.543 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:22:19.085 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_WARN Degraded data redundancy: 64/1647 objects degraded (3.886%), 3 pgs degraded 2024-04-22T15:22:19.085 INFO:teuthology.orchestra.run.smithi119.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 64/1647 objects degraded (3.886%), 3 pgs degraded 2024-04-22T15:22:19.085 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.4 is activating+undersized+degraded+remapped, acting [1,2] 2024-04-22T15:22:19.085 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.16 is activating+undersized+degraded+remapped, acting [1,7] 2024-04-22T15:22:19.085 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1f is activating+undersized+degraded+remapped, acting [0,2] 2024-04-22T15:22:19.086 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:18 smithi119 ceph-mon[63156]: from='client.44315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:19.086 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:18 smithi119 ceph-mon[63156]: from='client.44319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:19.086 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:18 smithi119 ceph-mon[63156]: osdmap e138: 8 total, 8 up, 8 in 2024-04-22T15:22:19.086 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:18 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/2304811018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:19.086 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:18 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 64/1602 objects degraded (3.995%), 3 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:19.086 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:18 smithi119 ceph-mon[63156]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:19.086 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:18 smithi119 ceph-mon[63156]: pgmap v246: 97 pgs: 3 peering, 3 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+remapped, 89 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 2.2 MiB/s wr, 168 op/s; 64/1647 objects degraded (3.886%); 1.3 MiB/s, 31 objects/s recovering 2024-04-22T15:22:19.201 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:18 smithi186 ceph-mon[49804]: from='client.44315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:19.201 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:18 smithi186 ceph-mon[49804]: from='client.44319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:19.201 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:18 smithi186 ceph-mon[49804]: osdmap e138: 8 total, 8 up, 8 in 2024-04-22T15:22:19.201 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:18 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/2304811018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:19.201 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:18 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 64/1602 objects degraded (3.995%), 3 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:19.201 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:18 smithi186 ceph-mon[49804]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:19.201 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:18 smithi186 ceph-mon[49804]: pgmap v246: 97 pgs: 3 peering, 3 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+remapped, 89 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 2.2 MiB/s wr, 168 op/s; 64/1647 objects degraded (3.886%); 1.3 MiB/s, 31 objects/s recovering 2024-04-22T15:22:19.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 270027 10.34 MB/sec execute 508 sec latency 64.355 ms 2024-04-22T15:22:19.857 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:19 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/598150079' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:22:20.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:19 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/598150079' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:22:20.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 270375 10.33 MB/sec execute 509 sec latency 49.435 ms 2024-04-22T15:22:20.876 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:20 smithi119 ceph-mon[63156]: pgmap v247: 97 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 93 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.7 MiB/s wr, 507 op/s; 23/1692 objects degraded (1.359%); 970 KiB/s, 28 objects/s recovering 2024-04-22T15:22:21.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:20 smithi186 ceph-mon[49804]: pgmap v247: 97 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 93 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.7 MiB/s wr, 507 op/s; 23/1692 objects degraded (1.359%); 970 KiB/s, 28 objects/s recovering 2024-04-22T15:22:21.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 270681 10.32 MB/sec execute 510 sec latency 51.515 ms 2024-04-22T15:22:22.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 271013 10.33 MB/sec execute 511 sec latency 44.321 ms 2024-04-22T15:22:22.902 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:22 smithi119 ceph-mon[63156]: pgmap v248: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.7 MiB/s wr, 389 op/s; 921 KiB/s, 23 objects/s recovering 2024-04-22T15:22:22.902 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:22 smithi119 ceph-mon[63156]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1692 objects degraded (1.359%), 1 pg degraded) 2024-04-22T15:22:22.902 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:22 smithi119 ceph-mon[63156]: Cluster is now healthy 2024-04-22T15:22:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:22 smithi186 ceph-mon[49804]: pgmap v248: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.7 MiB/s wr, 389 op/s; 921 KiB/s, 23 objects/s recovering 2024-04-22T15:22:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:22 smithi186 ceph-mon[49804]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1692 objects degraded (1.359%), 1 pg degraded) 2024-04-22T15:22:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:22 smithi186 ceph-mon[49804]: Cluster is now healthy 2024-04-22T15:22:23.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 271409 10.32 MB/sec execute 512 sec latency 59.284 ms 2024-04-22T15:22:24.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 271816 10.32 MB/sec execute 513 sec latency 51.949 ms 2024-04-22T15:22:24.678 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:22:25.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:24 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:22:25.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 272152 10.32 MB/sec execute 514 sec latency 51.037 ms 2024-04-22T15:22:25.969 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:25 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:22:25.969 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:25 smithi119 ceph-mon[63156]: Upgrade: osd.6 is safe to restart 2024-04-22T15:22:25.970 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:25 smithi119 ceph-mon[63156]: pgmap v249: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.3 MiB/s wr, 399 op/s; 764 KiB/s, 14 objects/s recovering 2024-04-22T15:22:25.970 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:25.970 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:25 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:22:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:25 smithi186 ceph-mon[49804]: Upgrade: osd.6 is safe to restart 2024-04-22T15:22:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:25 smithi186 ceph-mon[49804]: pgmap v249: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.3 MiB/s wr, 399 op/s; 764 KiB/s, 14 objects/s recovering 2024-04-22T15:22:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:26.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:26.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 272506 10.32 MB/sec execute 515 sec latency 64.611 ms 2024-04-22T15:22:27.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 272877 10.33 MB/sec execute 516 sec latency 69.359 ms 2024-04-22T15:22:27.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:27 smithi119 ceph-mon[63156]: pgmap v250: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 461 op/s; 614 KiB/s, 11 objects/s recovering 2024-04-22T15:22:27.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:27 smithi119 ceph-mon[63156]: Upgrade: Updating osd.6 2024-04-22T15:22:27.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:27 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:27.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:27 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:22:27.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:27 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:22:27.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:27 smithi119 ceph-mon[63156]: Deploying daemon osd.6 on smithi186 2024-04-22T15:22:28.063 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:27 smithi186 ceph-mon[49804]: pgmap v250: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 461 op/s; 614 KiB/s, 11 objects/s recovering 2024-04-22T15:22:28.063 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:27 smithi186 ceph-mon[49804]: Upgrade: Updating osd.6 2024-04-22T15:22:28.063 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:27 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:28.064 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:27 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:22:28.064 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:27 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:22:28.064 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:27 smithi186 ceph-mon[49804]: Deploying daemon osd.6 on smithi186 2024-04-22T15:22:28.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 273225 10.32 MB/sec execute 517 sec latency 43.521 ms 2024-04-22T15:22:29.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 273437 10.31 MB/sec execute 518 sec latency 60.737 ms 2024-04-22T15:22:29.995 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:29 smithi186 ceph-mon[49804]: pgmap v251: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.1 MiB/s wr, 544 op/s; 558 KiB/s, 10 objects/s recovering 2024-04-22T15:22:30.042 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:29 smithi119 ceph-mon[63156]: pgmap v251: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.1 MiB/s wr, 544 op/s; 558 KiB/s, 10 objects/s recovering 2024-04-22T15:22:30.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 273796 10.31 MB/sec execute 519 sec latency 70.681 ms 2024-04-22T15:22:31.046 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:30 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:31.046 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:30 smithi119 ceph-mon[63156]: pgmap v252: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 613 op/s; 437 KiB/s, 7 objects/s recovering 2024-04-22T15:22:31.181 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:30 smithi186 systemd[1]: Stopping Ceph osd.6 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:22:31.181 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:30 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:31.181 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:30 smithi186 ceph-mon[49804]: pgmap v252: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 613 op/s; 437 KiB/s, 7 objects/s recovering 2024-04-22T15:22:31.525 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:31 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[34211]: 2024-04-22T15:22:31.193+0000 7f3bc8e9f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:22:31.525 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:31 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[34211]: 2024-04-22T15:22:31.193+0000 7f3bc8e9f700 -1 osd.6 138 *** Got signal Terminated *** 2024-04-22T15:22:31.525 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:31 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[34211]: 2024-04-22T15:22:31.193+0000 7f3bc8e9f700 -1 osd.6 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:22:31.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 274255 10.31 MB/sec execute 520 sec latency 34.154 ms 2024-04-22T15:22:32.064 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:31 smithi119 ceph-mon[63156]: osd.6 marked itself down and dead 2024-04-22T15:22:32.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:31 smithi186 ceph-mon[49804]: osd.6 marked itself down and dead 2024-04-22T15:22:32.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 274573 10.32 MB/sec execute 521 sec latency 593.380 ms 2024-04-22T15:22:32.900 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:32 smithi186 podman[58871]: 2024-04-22 15:22:32.554005405 +0000 UTC m=+1.485302427 container died 2357de993a901a91799a55e4eb654d50f2b632628b491f2663d50d3f87edfbac (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6, GIT_CLEAN=True, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, distribution-scope=public, ceph=True, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-22T15:22:33.149 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:32 smithi119 ceph-mon[63156]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:22:33.149 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:32 smithi119 ceph-mon[63156]: osdmap e139: 8 total, 7 up, 8 in 2024-04-22T15:22:33.150 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:32 smithi119 ceph-mon[63156]: pgmap v254: 97 pgs: 10 stale+active+clean, 87 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 503 op/s 2024-04-22T15:22:33.213 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:32 smithi186 ceph-mon[49804]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:22:33.213 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:32 smithi186 ceph-mon[49804]: osdmap e139: 8 total, 7 up, 8 in 2024-04-22T15:22:33.213 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:32 smithi186 ceph-mon[49804]: pgmap v254: 97 pgs: 10 stale+active+clean, 87 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 503 op/s 2024-04-22T15:22:33.213 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:32 smithi186 podman[58871]: 2024-04-22 15:22:32.966989418 +0000 UTC m=+1.898286445 container cleanup 2357de993a901a91799a55e4eb654d50f2b632628b491f2663d50d3f87edfbac (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, ceph=True, io.openshift.expose-services=, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, release=754, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream) 2024-04-22T15:22:33.213 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:32 smithi186 bash[58871]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6 2024-04-22T15:22:33.525 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:33 smithi186 podman[58887]: 2024-04-22 15:22:33.210388668 +0000 UTC m=+0.660431136 container remove 2357de993a901a91799a55e4eb654d50f2b632628b491f2663d50d3f87edfbac (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, distribution-scope=public, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, release=754, name=centos-stream) 2024-04-22T15:22:33.527 INFO:teuthology.orchestra.run.smithi119.stdout: 5 274759 10.30 MB/sec execute 522 sec latency 1001.847 ms 2024-04-22T15:22:33.909 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:33 smithi119 ceph-mon[63156]: osdmap e140: 8 total, 7 up, 8 in 2024-04-22T15:22:33.992 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:33 smithi186 podman[58972]: 2024-04-22 15:22:33.608415728 +0000 UTC m=+0.019791340 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:22:33.992 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:33 smithi186 podman[58972]: 2024-04-22 15:22:33.736493009 +0000 UTC m=+0.147868612 container create 89689a1b6ed153ae9e652a1ad464c155fbc0d25b8f7cc10fd97901ba01017509 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:22:33.993 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:33 smithi186 ceph-mon[49804]: osdmap e140: 8 total, 7 up, 8 in 2024-04-22T15:22:34.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:33 smithi186 podman[58972]: 2024-04-22 15:22:33.98984489 +0000 UTC m=+0.401220512 container init 89689a1b6ed153ae9e652a1ad464c155fbc0d25b8f7cc10fd97901ba01017509 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-22T15:22:34.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:33 smithi186 podman[58972]: 2024-04-22 15:22:33.994551459 +0000 UTC m=+0.405927071 container start 89689a1b6ed153ae9e652a1ad464c155fbc0d25b8f7cc10fd97901ba01017509 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-04-22T15:22:34.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:34 smithi186 podman[58972]: 2024-04-22 15:22:34.089922787 +0000 UTC m=+0.501298402 container attach 89689a1b6ed153ae9e652a1ad464c155fbc0d25b8f7cc10fd97901ba01017509 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:22:34.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 275216 10.30 MB/sec execute 523 sec latency 57.689 ms 2024-04-22T15:22:34.742 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:34 smithi186 podman[58998]: 2024-04-22 15:22:34.367194801 +0000 UTC m=+0.028682654 container died 89689a1b6ed153ae9e652a1ad464c155fbc0d25b8f7cc10fd97901ba01017509 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-22T15:22:34.931 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:34 smithi119 ceph-mon[63156]: pgmap v256: 97 pgs: 1 active+undersized, 4 active+undersized+degraded, 18 peering, 3 stale+active+clean, 71 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.7 MiB/s wr, 558 op/s; 33/1689 objects degraded (1.954%) 2024-04-22T15:22:35.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:34 smithi186 ceph-mon[49804]: pgmap v256: 97 pgs: 1 active+undersized, 4 active+undersized+degraded, 18 peering, 3 stale+active+clean, 71 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.7 MiB/s wr, 558 op/s; 33/1689 objects degraded (1.954%) 2024-04-22T15:22:35.365 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:35 smithi186 podman[58998]: 2024-04-22 15:22:35.112102973 +0000 UTC m=+0.773590815 container remove 89689a1b6ed153ae9e652a1ad464c155fbc0d25b8f7cc10fd97901ba01017509 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux ) 2024-04-22T15:22:35.365 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:35 smithi186 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-22T15:22:35.365 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:35 smithi186 systemd[1]: Stopped Ceph osd.6 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:22:35.365 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:35 smithi186 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.6.service: Consumed 5min 20ms CPU time. 2024-04-22T15:22:35.365 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:35 smithi186 systemd[1]: Starting Ceph osd.6 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:22:35.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 275524 10.29 MB/sec execute 524 sec latency 58.197 ms 2024-04-22T15:22:35.894 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:35 smithi186 podman[59099]: 2024-04-22 15:22:35.500104621 +0000 UTC m=+0.020547479 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:22:35.894 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:35 smithi186 podman[59099]: 2024-04-22 15:22:35.622101293 +0000 UTC m=+0.142544139 container create f8668e6cc6da930f390a5e42376a5863c35871a51a1b2fe3de85cb0b3dfe51c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-22T15:22:35.947 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:35 smithi119 ceph-mon[63156]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T15:22:35.947 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:35 smithi119 ceph-mon[63156]: Health check failed: Degraded data redundancy: 33/1689 objects degraded (1.954%), 4 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:36.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:35 smithi186 ceph-mon[49804]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T15:22:36.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:35 smithi186 ceph-mon[49804]: Health check failed: Degraded data redundancy: 33/1689 objects degraded (1.954%), 4 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:36.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:35 smithi186 podman[59099]: 2024-04-22 15:22:35.891602402 +0000 UTC m=+0.412045248 container init f8668e6cc6da930f390a5e42376a5863c35871a51a1b2fe3de85cb0b3dfe51c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416) 2024-04-22T15:22:36.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:35 smithi186 podman[59099]: 2024-04-22 15:22:35.895197981 +0000 UTC m=+0.415640831 container start f8668e6cc6da930f390a5e42376a5863c35871a51a1b2fe3de85cb0b3dfe51c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-22T15:22:36.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:35 smithi186 podman[59099]: 2024-04-22 15:22:35.991874386 +0000 UTC m=+0.512317243 container attach f8668e6cc6da930f390a5e42376a5863c35871a51a1b2fe3de85cb0b3dfe51c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416) 2024-04-22T15:22:36.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 275885 10.30 MB/sec execute 525 sec latency 44.882 ms 2024-04-22T15:22:36.775 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate[59115]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:22:36.775 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate[59115]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 bash[59099]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 bash[59099]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 bash[59099]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 bash[59099]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 bash[59099]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 bash[59099]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 bash[59099]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 bash[59099]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 bash[59099]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate[59115]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate[59115]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate[59115]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate[59115]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T15:22:36.776 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate[59115]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-22T15:22:36.777 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate[59115]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T15:22:36.777 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate[59115]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-22T15:22:36.777 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:36 smithi186 podman[59099]: 2024-04-22 15:22:36.476282837 +0000 UTC m=+0.996725686 container died f8668e6cc6da930f390a5e42376a5863c35871a51a1b2fe3de85cb0b3dfe51c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-22T15:22:36.972 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:36 smithi119 ceph-mon[63156]: pgmap v257: 97 pgs: 3 active+undersized, 14 active+undersized+degraded, 18 peering, 62 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 427 op/s; 112/1725 objects degraded (6.493%) 2024-04-22T15:22:37.203 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:36 smithi186 ceph-mon[49804]: pgmap v257: 97 pgs: 3 active+undersized, 14 active+undersized+degraded, 18 peering, 62 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 427 op/s; 112/1725 objects degraded (6.493%) 2024-04-22T15:22:37.525 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:37 smithi186 podman[59253]: 2024-04-22 15:22:37.20107661 +0000 UTC m=+0.718940529 container remove f8668e6cc6da930f390a5e42376a5863c35871a51a1b2fe3de85cb0b3dfe51c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-activate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:22:37.525 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:37 smithi186 podman[59317]: 2024-04-22 15:22:37.320787794 +0000 UTC m=+0.018292412 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:22:37.525 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:37 smithi186 podman[59317]: 2024-04-22 15:22:37.431157056 +0000 UTC m=+0.128661673 container create 31e40cdb753c94695394c0a44bc57dd1ae5bafa454f24fcc7ad25cc22f4533e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:22:37.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 276281 10.30 MB/sec execute 526 sec latency 66.423 ms 2024-04-22T15:22:37.855 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:37 smithi186 podman[59317]: 2024-04-22 15:22:37.734080224 +0000 UTC m=+0.431584841 container init 31e40cdb753c94695394c0a44bc57dd1ae5bafa454f24fcc7ad25cc22f4533e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-22T15:22:37.855 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:37 smithi186 podman[59317]: 2024-04-22 15:22:37.737136049 +0000 UTC m=+0.434640668 container start 31e40cdb753c94695394c0a44bc57dd1ae5bafa454f24fcc7ad25cc22f4533e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_CLEAN=True) 2024-04-22T15:22:37.856 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:37 smithi186 ceph-osd[59336]: -- 172.21.15.186:0/1837183876 <== mon.1 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c9fd2a9680 con 0x55c9fd272c00 2024-04-22T15:22:38.278 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:37 smithi186 bash[59317]: 31e40cdb753c94695394c0a44bc57dd1ae5bafa454f24fcc7ad25cc22f4533e9 2024-04-22T15:22:38.278 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:37 smithi186 systemd[1]: Started Ceph osd.6 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:22:38.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 276682 10.31 MB/sec execute 527 sec latency 47.878 ms 2024-04-22T15:22:38.775 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:38 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:22:38.278+0000 7fabdfbd07c0 -1 Falling back to public interface 2024-04-22T15:22:39.013 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:39.013 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:39.013 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:22:39.013 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:38 smithi119 ceph-mon[63156]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.6 MiB/s wr, 498 op/s; 193/1689 objects degraded (11.427%) 2024-04-22T15:22:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:38 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:38 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:38 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:22:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:38 smithi186 ceph-mon[49804]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.6 MiB/s wr, 498 op/s; 193/1689 objects degraded (11.427%) 2024-04-22T15:22:39.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 277028 10.30 MB/sec execute 528 sec latency 55.495 ms 2024-04-22T15:22:40.042 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:39 smithi119 ceph-mon[63156]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T15:22:40.043 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:40.043 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:40.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:39 smithi186 ceph-mon[49804]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T15:22:40.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:39 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:40.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:39 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:40.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 277354 10.29 MB/sec execute 529 sec latency 54.188 ms 2024-04-22T15:22:41.074 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:40 smithi119 ceph-mon[63156]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 5.1 MiB/s wr, 479 op/s; 187/1683 objects degraded (11.111%) 2024-04-22T15:22:41.254 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:40 smithi186 ceph-mon[49804]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 5.1 MiB/s wr, 479 op/s; 187/1683 objects degraded (11.111%) 2024-04-22T15:22:41.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 277687 10.29 MB/sec execute 530 sec latency 59.030 ms 2024-04-22T15:22:42.107 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:41 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 187/1683 objects degraded (11.111%), 25 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:42.107 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:41 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:42.107 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:41 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:42.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:41 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 187/1683 objects degraded (11.111%), 25 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:42.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:41 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:42.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:41 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:42.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 278002 10.29 MB/sec execute 531 sec latency 46.593 ms 2024-04-22T15:22:43.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:42 smithi119 ceph-mon[63156]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 438 op/s; 185/1674 objects degraded (11.051%) 2024-04-22T15:22:43.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:42 smithi186 ceph-mon[49804]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 438 op/s; 185/1674 objects degraded (11.051%) 2024-04-22T15:22:43.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 278338 10.29 MB/sec execute 532 sec latency 49.409 ms 2024-04-22T15:22:44.134 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:44 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:44.134 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:44 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:44.169 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:44 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:44.169 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:44 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:44.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 278759 10.29 MB/sec execute 533 sec latency 51.666 ms 2024-04-22T15:22:45.147 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:45 smithi119 ceph-mon[63156]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.7 MiB/s wr, 472 op/s; 192/1677 objects degraded (11.449%) 2024-04-22T15:22:45.235 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:45 smithi186 ceph-mon[49804]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.7 MiB/s wr, 472 op/s; 192/1677 objects degraded (11.449%) 2024-04-22T15:22:45.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 279066 10.28 MB/sec execute 534 sec latency 30.800 ms 2024-04-22T15:22:45.775 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:45 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:22:45.748+0000 7fabdfbd07c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-22T15:22:46.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 279387 10.28 MB/sec execute 535 sec latency 42.261 ms 2024-04-22T15:22:46.927 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:46 smithi119 ceph-mon[63156]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.0 MiB/s wr, 476 op/s; 199/1710 objects degraded (11.637%) 2024-04-22T15:22:46.928 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:46 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 199/1710 objects degraded (11.637%), 25 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:47.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:46 smithi186 ceph-mon[49804]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.0 MiB/s wr, 476 op/s; 199/1710 objects degraded (11.637%) 2024-04-22T15:22:47.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:46 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 199/1710 objects degraded (11.637%), 25 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:47.276 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:47 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:22:47.124+0000 7fabdfbd07c0 -1 osd.6 138 log_to_monitors true 2024-04-22T15:22:47.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 279716 10.28 MB/sec execute 536 sec latency 52.854 ms 2024-04-22T15:22:48.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:48.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:48.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:22:48.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:22:48.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:48.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:48.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='osd.6 [v2:172.21.15.186:6816/2244741745,v1:172.21.15.186:6817/2244741745]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:22:48.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:22:48.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:22:48.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:48.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:48.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:48.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:22:48.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:22:48.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:48 smithi119 ceph-mon[63156]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-22T15:22:48.320 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='osd.6 [v2:172.21.15.186:6816/2244741745,v1:172.21.15.186:6817/2244741745]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:22:48.321 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-mon[49804]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-22T15:22:48.528 INFO:teuthology.orchestra.run.smithi119.stdout: 5 280004 10.28 MB/sec execute 537 sec latency 120.017 ms 2024-04-22T15:22:48.775 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:22:48 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:22:48.562+0000 7fabcd020700 -1 osd.6 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:22:49.291 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:49 smithi119 ceph-mon[63156]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T15:22:49.291 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:49 smithi119 ceph-mon[63156]: osdmap e141: 8 total, 7 up, 8 in 2024-04-22T15:22:49.292 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:49 smithi119 ceph-mon[63156]: from='osd.6 [v2:172.21.15.186:6816/2244741745,v1:172.21.15.186:6817/2244741745]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:22:49.292 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:49 smithi119 ceph-mon[63156]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:22:49.292 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:49 smithi119 ceph-mon[63156]: from='osd.6 ' entity='osd.6' 2024-04-22T15:22:49.292 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:49 smithi119 ceph-mon[63156]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 352 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.9 MiB/s wr, 462 op/s; 191/1683 objects degraded (11.349%) 2024-04-22T15:22:49.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:49 smithi186 ceph-mon[49804]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T15:22:49.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:49 smithi186 ceph-mon[49804]: osdmap e141: 8 total, 7 up, 8 in 2024-04-22T15:22:49.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:49 smithi186 ceph-mon[49804]: from='osd.6 [v2:172.21.15.186:6816/2244741745,v1:172.21.15.186:6817/2244741745]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:22:49.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:49 smithi186 ceph-mon[49804]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:22:49.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:49 smithi186 ceph-mon[49804]: from='osd.6 ' entity='osd.6' 2024-04-22T15:22:49.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:49 smithi186 ceph-mon[49804]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 352 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.9 MiB/s wr, 462 op/s; 191/1683 objects degraded (11.349%) 2024-04-22T15:22:49.533 INFO:teuthology.orchestra.run.smithi119.stdout: 5 280316 10.28 MB/sec execute 538 sec latency 61.021 ms 2024-04-22T15:22:49.570 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:22:50.257 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:22:50.257 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (8m) 2m ago 17m 15.0M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:22:50.257 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (5m) 2m ago 17m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (5m) 8s ago 16m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (8m) 2m ago 17m 47.8M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (11m) 2m ago 11m 38.4M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (11m) 8s ago 11m 15.1M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (7m) 2m ago 18m 509M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (6m) 8s ago 16m 452M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (6m) 2m ago 18m 52.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (6m) 8s ago 16m 46.6M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (11m) 2m ago 11m 285M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (11m) 8s ago 11m 84.9M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (8m) 2m ago 17m 10.4M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (8m) 8s ago 16m 18.0M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (4m) 2m ago 15m 357M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:22:50.258 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (3m) 2m ago 15m 286M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:22:50.259 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (2m) 2m ago 15m 220M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:22:50.259 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (2m) 2m ago 14m 12.8M 1936M 19.0.0-3223-g5883d494 3d983c26e278 2378efa46eca 2024-04-22T15:22:50.259 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (87s) 8s ago 14m 297M 2704M 19.0.0-3223-g5883d494 3d983c26e278 9518e2eb7665 2024-04-22T15:22:50.259 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (50s) 8s ago 13m 277M 2704M 19.0.0-3223-g5883d494 3d983c26e278 e1c8f1367a89 2024-04-22T15:22:50.259 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (12s) 8s ago 13m 12.0M 2704M 19.0.0-3223-g5883d494 3d983c26e278 31e40cdb753c 2024-04-22T15:22:50.259 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (13m) 8s ago 13m 650M 2704M 17.2.0 e1d6a67b021e 8d207088ee8a 2024-04-22T15:22:50.259 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (7m) 2m ago 16m 68.0M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:22:50.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:50 smithi119 ceph-mon[63156]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:50.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:50 smithi119 ceph-mon[63156]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:22:50.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:50 smithi119 ceph-mon[63156]: osd.6 [v2:172.21.15.186:6816/2244741745,v1:172.21.15.186:6817/2244741745] boot 2024-04-22T15:22:50.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:50 smithi119 ceph-mon[63156]: osdmap e142: 8 total, 8 up, 8 in 2024-04-22T15:22:50.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:50 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:22:50.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:50 smithi186 ceph-mon[49804]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:50.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:50 smithi186 ceph-mon[49804]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:22:50.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:50 smithi186 ceph-mon[49804]: osd.6 [v2:172.21.15.186:6816/2244741745,v1:172.21.15.186:6817/2244741745] boot 2024-04-22T15:22:50.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:50 smithi186 ceph-mon[49804]: osdmap e142: 8 total, 8 up, 8 in 2024-04-22T15:22:50.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:50 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:22:50.533 INFO:teuthology.orchestra.run.smithi119.stdout: 5 280332 10.26 MB/sec execute 539 sec latency 950.376 ms 2024-04-22T15:22:50.690 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:22:50.690 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:22:50.690 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:22:50.690 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:22:50.690 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:22:50.690 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 7 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 11 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:22:50.691 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:22:51.035 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:22:51.035 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:22:51.036 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:22:51.036 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:22:51.036 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:22:51.036 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:22:51.036 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:22:51.036 INFO:teuthology.orchestra.run.smithi119.stdout: "crash" 2024-04-22T15:22:51.036 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:22:51.036 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "13/23 daemons upgraded", 2024-04-22T15:22:51.036 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:22:51.036 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:22:51.036 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:22:51.454 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_WARN Degraded data redundancy: 191/1683 objects degraded (11.349%), 25 pgs degraded 2024-04-22T15:22:51.454 INFO:teuthology.orchestra.run.smithi119.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 191/1683 objects degraded (11.349%), 25 pgs degraded 2024-04-22T15:22:51.454 INFO:teuthology.orchestra.run.smithi119.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-04-22T15:22:51.454 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-04-22T15:22:51.454 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-04-22T15:22:51.454 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-04-22T15:22:51.454 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-04-22T15:22:51.455 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-04-22T15:22:51.456 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-04-22T15:22:51.456 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-04-22T15:22:51.456 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-04-22T15:22:51.456 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-04-22T15:22:51.456 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-04-22T15:22:51.456 INFO:teuthology.orchestra.run.smithi119.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-04-22T15:22:51.456 INFO:teuthology.orchestra.run.smithi119.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-04-22T15:22:51.533 INFO:teuthology.orchestra.run.smithi119.stdout: 5 280332 10.25 MB/sec execute 540 sec latency 1950.441 ms 2024-04-22T15:22:51.710 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:51 smithi119 ceph-mon[63156]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:51.710 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:51 smithi119 ceph-mon[63156]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:51.711 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:51 smithi119 ceph-mon[63156]: osdmap e143: 8 total, 8 up, 8 in 2024-04-22T15:22:51.711 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:51 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/2922169089' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:51.711 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:51 smithi119 ceph-mon[63156]: pgmap v267: 97 pgs: 1 active+recovering+degraded, 2 remapped+peering, 10 active+undersized, 21 active+undersized+degraded, 63 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.3 MiB/s wr, 508 op/s; 170/1665 objects degraded (10.210%); 0 B/s, 0 objects/s recovering 2024-04-22T15:22:51.711 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:51 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/2121058539' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:22:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:51 smithi186 ceph-mon[49804]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:51 smithi186 ceph-mon[49804]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:51 smithi186 ceph-mon[49804]: osdmap e143: 8 total, 8 up, 8 in 2024-04-22T15:22:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:51 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/2922169089' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:22:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:51 smithi186 ceph-mon[49804]: pgmap v267: 97 pgs: 1 active+recovering+degraded, 2 remapped+peering, 10 active+undersized, 21 active+undersized+degraded, 63 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.3 MiB/s wr, 508 op/s; 170/1665 objects degraded (10.210%); 0 B/s, 0 objects/s recovering 2024-04-22T15:22:52.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:51 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/2121058539' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:22:52.533 INFO:teuthology.orchestra.run.smithi119.stdout: 5 280590 10.23 MB/sec execute 541 sec latency 2065.156 ms 2024-04-22T15:22:52.714 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:52 smithi119 ceph-mon[63156]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:52.714 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:52 smithi119 ceph-mon[63156]: osdmap e144: 8 total, 8 up, 8 in 2024-04-22T15:22:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:52 smithi186 ceph-mon[49804]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:22:53.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:52 smithi186 ceph-mon[49804]: osdmap e144: 8 total, 8 up, 8 in 2024-04-22T15:22:53.533 INFO:teuthology.orchestra.run.smithi119.stdout: 5 280812 10.22 MB/sec execute 542 sec latency 865.321 ms 2024-04-22T15:22:53.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:53 smithi119 ceph-mon[63156]: osdmap e145: 8 total, 8 up, 8 in 2024-04-22T15:22:53.682 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:53 smithi119 ceph-mon[63156]: pgmap v270: 97 pgs: 1 active+recovering+degraded, 3 remapped+peering, 9 active+undersized, 17 active+undersized+degraded, 67 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s wr, 345 op/s; 134/1641 objects degraded (8.166%); 25 KiB/s, 5 objects/s recovering 2024-04-22T15:22:53.683 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:53 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 170/1665 objects degraded (10.210%), 22 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:54.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:53 smithi186 ceph-mon[49804]: osdmap e145: 8 total, 8 up, 8 in 2024-04-22T15:22:54.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:53 smithi186 ceph-mon[49804]: pgmap v270: 97 pgs: 1 active+recovering+degraded, 3 remapped+peering, 9 active+undersized, 17 active+undersized+degraded, 67 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s wr, 345 op/s; 134/1641 objects degraded (8.166%); 25 KiB/s, 5 objects/s recovering 2024-04-22T15:22:54.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:53 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 170/1665 objects degraded (10.210%), 22 pgs degraded (PG_DEGRADED) 2024-04-22T15:22:54.533 INFO:teuthology.orchestra.run.smithi119.stdout: 5 281148 10.21 MB/sec execute 543 sec latency 957.354 ms 2024-04-22T15:22:54.710 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:54 smithi119 ceph-mon[63156]: osdmap e146: 8 total, 8 up, 8 in 2024-04-22T15:22:55.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:54 smithi186 ceph-mon[49804]: osdmap e146: 8 total, 8 up, 8 in 2024-04-22T15:22:55.533 INFO:teuthology.orchestra.run.smithi119.stdout: 5 281422 10.21 MB/sec execute 544 sec latency 45.742 ms 2024-04-22T15:22:55.976 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:55.976 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:55 smithi119 ceph-mon[63156]: pgmap v272: 97 pgs: 5 peering, 1 active+recovering+degraded, 3 remapped+peering, 88 active+clean; 348 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.5 MiB/s wr, 450 op/s; 5/1623 objects degraded (0.308%); 4.7 MiB/s, 63 keys/s, 41 objects/s recovering 2024-04-22T15:22:55.976 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:22:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:55 smithi186 ceph-mon[49804]: pgmap v272: 97 pgs: 5 peering, 1 active+recovering+degraded, 3 remapped+peering, 88 active+clean; 348 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.5 MiB/s wr, 450 op/s; 5/1623 objects degraded (0.308%); 4.7 MiB/s, 63 keys/s, 41 objects/s recovering 2024-04-22T15:22:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:56.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 281886 10.22 MB/sec execute 545 sec latency 49.329 ms 2024-04-22T15:22:57.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:56 smithi119 ceph-mon[63156]: pgmap v273: 97 pgs: 5 peering, 92 active+clean; 345 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.4 MiB/s wr, 210 op/s; 3.9 MiB/s, 44 keys/s, 40 objects/s recovering 2024-04-22T15:22:57.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:56 smithi119 ceph-mon[63156]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/1623 objects degraded (0.308%), 1 pg degraded) 2024-04-22T15:22:57.008 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:56 smithi119 ceph-mon[63156]: Cluster is now healthy 2024-04-22T15:22:57.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:56 smithi186 ceph-mon[49804]: pgmap v273: 97 pgs: 5 peering, 92 active+clean; 345 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.4 MiB/s wr, 210 op/s; 3.9 MiB/s, 44 keys/s, 40 objects/s recovering 2024-04-22T15:22:57.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:56 smithi186 ceph-mon[49804]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/1623 objects degraded (0.308%), 1 pg degraded) 2024-04-22T15:22:57.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:56 smithi186 ceph-mon[49804]: Cluster is now healthy 2024-04-22T15:22:57.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 282320 10.22 MB/sec execute 546 sec latency 35.010 ms 2024-04-22T15:22:58.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 282591 10.21 MB/sec execute 547 sec latency 44.341 ms 2024-04-22T15:22:59.040 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:22:58 smithi119 ceph-mon[63156]: pgmap v274: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 3.1 MiB/s wr, 320 op/s; 3.2 MiB/s, 37 keys/s, 34 objects/s recovering 2024-04-22T15:22:59.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:22:58 smithi186 ceph-mon[49804]: pgmap v274: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 3.1 MiB/s wr, 320 op/s; 3.2 MiB/s, 37 keys/s, 34 objects/s recovering 2024-04-22T15:22:59.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 282944 10.22 MB/sec execute 548 sec latency 44.712 ms 2024-04-22T15:23:00.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 283302 10.21 MB/sec execute 549 sec latency 47.312 ms 2024-04-22T15:23:01.085 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:00 smithi119 ceph-mon[63156]: pgmap v275: 97 pgs: 97 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.4 MiB/s wr, 410 op/s; 2.8 MiB/s, 32 keys/s, 30 objects/s recovering 2024-04-22T15:23:01.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:00 smithi186 ceph-mon[49804]: pgmap v275: 97 pgs: 97 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.4 MiB/s wr, 410 op/s; 2.8 MiB/s, 32 keys/s, 30 objects/s recovering 2024-04-22T15:23:01.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 283653 10.22 MB/sec execute 550 sec latency 41.224 ms 2024-04-22T15:23:02.360 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:02 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:23:02.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 283918 10.21 MB/sec execute 551 sec latency 51.531 ms 2024-04-22T15:23:02.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:02 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:23:03.377 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:03 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:23:03.378 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:03 smithi119 ceph-mon[63156]: Upgrade: osd.7 is safe to restart 2024-04-22T15:23:03.378 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:03 smithi119 ceph-mon[63156]: pgmap v276: 97 pgs: 97 active+clean; 351 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.8 MiB/s wr, 347 op/s; 2.3 MiB/s, 26 keys/s, 24 objects/s recovering 2024-04-22T15:23:03.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:03 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:23:03.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:03 smithi186 ceph-mon[49804]: Upgrade: osd.7 is safe to restart 2024-04-22T15:23:03.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:03 smithi186 ceph-mon[49804]: pgmap v276: 97 pgs: 97 active+clean; 351 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.8 MiB/s wr, 347 op/s; 2.3 MiB/s, 26 keys/s, 24 objects/s recovering 2024-04-22T15:23:03.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 284195 10.20 MB/sec execute 552 sec latency 55.196 ms 2024-04-22T15:23:04.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 284492 10.19 MB/sec execute 553 sec latency 56.000 ms 2024-04-22T15:23:05.167 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:05 smithi119 ceph-mon[63156]: Upgrade: Updating osd.7 2024-04-22T15:23:05.168 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:05 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:05.168 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:05 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:23:05.168 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:05 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:23:05.168 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:05 smithi119 ceph-mon[63156]: Deploying daemon osd.7 on smithi186 2024-04-22T15:23:05.168 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:05 smithi119 ceph-mon[63156]: pgmap v277: 97 pgs: 97 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 469 op/s; 389 KiB/s, 23 keys/s, 11 objects/s recovering 2024-04-22T15:23:05.168 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:05 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:05.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:05 smithi186 ceph-mon[49804]: Upgrade: Updating osd.7 2024-04-22T15:23:05.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:05 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:05.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:05 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:23:05.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:05 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:23:05.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:05 smithi186 ceph-mon[49804]: Deploying daemon osd.7 on smithi186 2024-04-22T15:23:05.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:05 smithi186 ceph-mon[49804]: pgmap v277: 97 pgs: 97 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 469 op/s; 389 KiB/s, 23 keys/s, 11 objects/s recovering 2024-04-22T15:23:05.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:05 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:05.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 284744 10.19 MB/sec execute 554 sec latency 64.204 ms 2024-04-22T15:23:06.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 284988 10.18 MB/sec execute 555 sec latency 74.823 ms 2024-04-22T15:23:06.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:06 smithi119 ceph-mon[63156]: pgmap v278: 97 pgs: 97 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 486 op/s; 314 KiB/s, 8 objects/s recovering 2024-04-22T15:23:07.154 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:06 smithi186 ceph-mon[49804]: pgmap v278: 97 pgs: 97 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 486 op/s; 314 KiB/s, 8 objects/s recovering 2024-04-22T15:23:07.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 285337 10.19 MB/sec execute 556 sec latency 77.491 ms 2024-04-22T15:23:08.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:07 smithi186 systemd[1]: Stopping Ceph osd.7 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:23:08.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:08 smithi119 ceph-mon[63156]: osd.7 marked itself down and dead 2024-04-22T15:23:08.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:08 smithi119 ceph-mon[63156]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:23:08.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:08 smithi186 ceph-mon[49804]: osd.7 marked itself down and dead 2024-04-22T15:23:08.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:08 smithi186 ceph-mon[49804]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:23:08.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:08 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[37219]: 2024-04-22T15:23:08.108+0000 7f3541687700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:23:08.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:08 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[37219]: 2024-04-22T15:23:08.108+0000 7f3541687700 -1 osd.7 146 *** Got signal Terminated *** 2024-04-22T15:23:08.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:08 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[37219]: 2024-04-22T15:23:08.108+0000 7f3541687700 -1 osd.7 146 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:23:08.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 285536 10.18 MB/sec execute 557 sec latency 386.023 ms 2024-04-22T15:23:09.160 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:08 smithi186 podman[61154]: 2024-04-22 15:23:08.863106693 +0000 UTC m=+0.885669568 container died 8d207088ee8aa121ed1ad2949034967b6fa3c80dc8cf1e91eb7cfde95123b995 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7, RELEASE=HEAD, GIT_CLEAN=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, architecture=x86_64, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-22T15:23:09.434 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:09 smithi119 ceph-mon[63156]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-22T15:23:09.434 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:09 smithi119 ceph-mon[63156]: osdmap e147: 8 total, 7 up, 8 in 2024-04-22T15:23:09.435 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:09 smithi119 ceph-mon[63156]: pgmap v280: 97 pgs: 13 stale+active+clean, 84 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 524 op/s 2024-04-22T15:23:09.435 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:09 smithi119 ceph-mon[63156]: osdmap e148: 8 total, 7 up, 8 in 2024-04-22T15:23:09.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:09 smithi186 ceph-mon[49804]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-22T15:23:09.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:09 smithi186 ceph-mon[49804]: osdmap e147: 8 total, 7 up, 8 in 2024-04-22T15:23:09.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:09 smithi186 ceph-mon[49804]: pgmap v280: 97 pgs: 13 stale+active+clean, 84 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 524 op/s 2024-04-22T15:23:09.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:09 smithi186 ceph-mon[49804]: osdmap e148: 8 total, 7 up, 8 in 2024-04-22T15:23:09.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:09 smithi186 podman[61154]: 2024-04-22 15:23:09.257522243 +0000 UTC m=+1.280085133 container cleanup 8d207088ee8aa121ed1ad2949034967b6fa3c80dc8cf1e91eb7cfde95123b995 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, release=754, name=centos-stream, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, RELEASE=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, architecture=x86_64, distribution-scope=public, GIT_BRANCH=HEAD) 2024-04-22T15:23:09.526 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:09 smithi186 bash[61154]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7 2024-04-22T15:23:09.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 285674 10.16 MB/sec execute 558 sec latency 1032.523 ms 2024-04-22T15:23:09.958 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:09 smithi186 podman[61171]: 2024-04-22 15:23:09.538391173 +0000 UTC m=+0.671286023 container remove 8d207088ee8aa121ed1ad2949034967b6fa3c80dc8cf1e91eb7cfde95123b995 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7, GIT_BRANCH=HEAD, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, GIT_CLEAN=True, name=centos-stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.openshift.expose-services=, release=754, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, CEPH_POINT_RELEASE=-17.2.0) 2024-04-22T15:23:10.263 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:09 smithi186 podman[61255]: 2024-04-22 15:23:09.859568847 +0000 UTC m=+0.022684466 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:23:10.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:10 smithi186 podman[61255]: 2024-04-22 15:23:10.260445914 +0000 UTC m=+0.423561552 container create 7a5cf7eb33977a1bc4b5107d2acb2f0a499e2731a02e91e06c651ea2647ad807 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True) 2024-04-22T15:23:10.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:10 smithi186 podman[61255]: 2024-04-22 15:23:10.497587158 +0000 UTC m=+0.660702768 container init 7a5cf7eb33977a1bc4b5107d2acb2f0a499e2731a02e91e06c651ea2647ad807 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:23:10.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:10 smithi186 podman[61255]: 2024-04-22 15:23:10.502690166 +0000 UTC m=+0.665805761 container start 7a5cf7eb33977a1bc4b5107d2acb2f0a499e2731a02e91e06c651ea2647ad807 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True) 2024-04-22T15:23:10.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 286046 10.16 MB/sec execute 559 sec latency 65.282 ms 2024-04-22T15:23:10.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:10.971 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:11.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:11.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:10 smithi186 podman[61255]: 2024-04-22 15:23:10.639087799 +0000 UTC m=+0.802203397 container attach 7a5cf7eb33977a1bc4b5107d2acb2f0a499e2731a02e91e06c651ea2647ad807 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-22T15:23:11.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:10 smithi186 podman[61255]: 2024-04-22 15:23:10.874188026 +0000 UTC m=+1.037303627 container died 7a5cf7eb33977a1bc4b5107d2acb2f0a499e2731a02e91e06c651ea2647ad807 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True) 2024-04-22T15:23:11.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:11 smithi186 podman[61280]: 2024-04-22 15:23:11.440170262 +0000 UTC m=+0.559406159 container remove 7a5cf7eb33977a1bc4b5107d2acb2f0a499e2731a02e91e06c651ea2647ad807 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-22T15:23:11.526 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:11 smithi186 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.7.service: Deactivated successfully. 2024-04-22T15:23:11.526 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:11 smithi186 systemd[1]: Stopped Ceph osd.7 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:23:11.526 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:11 smithi186 systemd[1]: ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.7.service: Consumed 7min 43.739s CPU time. 2024-04-22T15:23:11.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 286373 10.16 MB/sec execute 560 sec latency 37.436 ms 2024-04-22T15:23:11.989 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:11 smithi119 ceph-mon[63156]: pgmap v282: 97 pgs: 1 active+undersized+degraded, 13 peering, 9 stale+active+clean, 74 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.6 MiB/s wr, 535 op/s; 8/1662 objects degraded (0.481%) 2024-04-22T15:23:11.997 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:11 smithi186 systemd[1]: Starting Ceph osd.7 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:23:11.997 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:11 smithi186 podman[61378]: 2024-04-22 15:23:11.833049991 +0000 UTC m=+0.021188919 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:23:11.998 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:11 smithi186 ceph-mon[49804]: pgmap v282: 97 pgs: 1 active+undersized+degraded, 13 peering, 9 stale+active+clean, 74 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.6 MiB/s wr, 535 op/s; 8/1662 objects degraded (0.481%) 2024-04-22T15:23:12.251 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:11 smithi186 podman[61378]: 2024-04-22 15:23:11.995337921 +0000 UTC m=+0.183476832 container create 10a5b735b40c0c63989129429a1ee2c465bd30ec4438a6ed417da268f06abb0a (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-22T15:23:12.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 podman[61378]: 2024-04-22 15:23:12.248350213 +0000 UTC m=+0.436489125 container init 10a5b735b40c0c63989129429a1ee2c465bd30ec4438a6ed417da268f06abb0a (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416) 2024-04-22T15:23:12.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 podman[61378]: 2024-04-22 15:23:12.251902519 +0000 UTC m=+0.440041435 container start 10a5b735b40c0c63989129429a1ee2c465bd30ec4438a6ed417da268f06abb0a (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-22T15:23:12.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 podman[61378]: 2024-04-22 15:23:12.348456953 +0000 UTC m=+0.536595869 container attach 10a5b735b40c0c63989129429a1ee2c465bd30ec4438a6ed417da268f06abb0a (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5) 2024-04-22T15:23:12.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 286757 10.16 MB/sec execute 561 sec latency 37.950 ms 2024-04-22T15:23:13.019 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:12 smithi119 ceph-mon[63156]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-22T15:23:13.019 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:12 smithi119 ceph-mon[63156]: Health check failed: Degraded data redundancy: 8/1662 objects degraded (0.481%), 1 pg degraded (PG_DEGRADED) 2024-04-22T15:23:13.205 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:12 smithi186 ceph-mon[49804]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-22T15:23:13.205 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:12 smithi186 ceph-mon[49804]: Health check failed: Degraded data redundancy: 8/1662 objects degraded (0.481%), 1 pg degraded (PG_DEGRADED) 2024-04-22T15:23:13.206 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate[61393]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:23:13.206 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate[61393]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:23:13.206 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 bash[61378]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:23:13.206 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 bash[61378]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:23:13.206 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 bash[61378]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T15:23:13.206 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 bash[61378]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T15:23:13.206 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 bash[61378]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T15:23:13.206 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 bash[61378]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T15:23:13.206 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 bash[61378]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-22T15:23:13.206 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 bash[61378]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T15:23:13.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 bash[61378]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-22T15:23:13.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate[61393]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T15:23:13.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate[61393]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T15:23:13.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate[61393]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T15:23:13.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate[61393]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T15:23:13.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate[61393]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-22T15:23:13.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate[61393]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T15:23:13.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate[61393]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-22T15:23:13.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:12 smithi186 podman[61378]: 2024-04-22 15:23:12.889217637 +0000 UTC m=+1.077356556 container died 10a5b735b40c0c63989129429a1ee2c465bd30ec4438a6ed417da268f06abb0a (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5) 2024-04-22T15:23:13.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 287134 10.17 MB/sec execute 562 sec latency 54.589 ms 2024-04-22T15:23:13.765 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:13 smithi186 podman[61521]: 2024-04-22 15:23:13.495457787 +0000 UTC m=+0.600288753 container remove 10a5b735b40c0c63989129429a1ee2c465bd30ec4438a6ed417da268f06abb0a (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-22T15:23:13.765 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:13 smithi186 podman[61578]: 2024-04-22 15:23:13.634153169 +0000 UTC m=+0.021236021 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:23:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:13 smithi186 ceph-mon[49804]: pgmap v283: 97 pgs: 2 active+undersized, 7 active+undersized+degraded, 13 peering, 7 stale+active+clean, 68 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s wr, 310 op/s; 48/1641 objects degraded (2.925%) 2024-04-22T15:23:14.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:13 smithi186 podman[61578]: 2024-04-22 15:23:13.763093526 +0000 UTC m=+0.150176361 container create 2f6a10f5d68af3ce971206a5eb05cb4babf3e1b2ebe05907fa6fb639d29d7aae (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-22T15:23:14.048 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:13 smithi119 ceph-mon[63156]: pgmap v283: 97 pgs: 2 active+undersized, 7 active+undersized+degraded, 13 peering, 7 stale+active+clean, 68 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s wr, 310 op/s; 48/1641 objects degraded (2.925%) 2024-04-22T15:23:14.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:14 smithi186 podman[61578]: 2024-04-22 15:23:14.199460962 +0000 UTC m=+0.586543789 container init 2f6a10f5d68af3ce971206a5eb05cb4babf3e1b2ebe05907fa6fb639d29d7aae (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-22T15:23:14.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:14 smithi186 podman[61578]: 2024-04-22 15:23:14.202736371 +0000 UTC m=+0.589819205 container start 2f6a10f5d68af3ce971206a5eb05cb4babf3e1b2ebe05907fa6fb639d29d7aae (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416) 2024-04-22T15:23:14.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:14 smithi186 ceph-osd[61597]: -- 172.21.15.186:0/1750864219 <== mon.1 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563258b59680 con 0x563258b22c00 2024-04-22T15:23:14.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:14 smithi186 bash[61578]: 2f6a10f5d68af3ce971206a5eb05cb4babf3e1b2ebe05907fa6fb639d29d7aae 2024-04-22T15:23:14.525 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:14 smithi186 systemd[1]: Started Ceph osd.7 for 74739db6-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:23:14.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 287435 10.16 MB/sec execute 563 sec latency 54.119 ms 2024-04-22T15:23:15.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:14 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:23:14.739+0000 7f6d6064e7c0 -1 Falling back to public interface 2024-04-22T15:23:15.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 287767 10.16 MB/sec execute 564 sec latency 48.701 ms 2024-04-22T15:23:15.570 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:15 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:15.570 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:15 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:15.570 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:15 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:23:15.570 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:15 smithi119 ceph-mon[63156]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 354 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 415 op/s; 273/1689 objects degraded (16.163%) 2024-04-22T15:23:15.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:15 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:15.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:15 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:15.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:15 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:23:15.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:15 smithi186 ceph-mon[49804]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 354 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 415 op/s; 273/1689 objects degraded (16.163%) 2024-04-22T15:23:16.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:16 smithi186 ceph-mon[49804]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-04-22T15:23:16.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 288141 10.15 MB/sec execute 565 sec latency 51.259 ms 2024-04-22T15:23:16.583 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:16 smithi119 ceph-mon[63156]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-04-22T15:23:17.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 288399 10.14 MB/sec execute 566 sec latency 42.252 ms 2024-04-22T15:23:17.595 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:17 smithi119 ceph-mon[63156]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s wr, 292 op/s; 275/1695 objects degraded (16.224%) 2024-04-22T15:23:17.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:17 smithi186 ceph-mon[49804]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s wr, 292 op/s; 275/1695 objects degraded (16.224%) 2024-04-22T15:23:18.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 288759 10.15 MB/sec execute 567 sec latency 48.341 ms 2024-04-22T15:23:18.619 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:18 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 275/1695 objects degraded (16.224%), 35 pgs degraded (PG_DEGRADED) 2024-04-22T15:23:18.619 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:18 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:18.619 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:18 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:18.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:18 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 275/1695 objects degraded (16.224%), 35 pgs degraded (PG_DEGRADED) 2024-04-22T15:23:18.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:18 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:18.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:18 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:19.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 289107 10.15 MB/sec execute 568 sec latency 69.811 ms 2024-04-22T15:23:19.642 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:19 smithi119 ceph-mon[63156]: pgmap v286: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.9 MiB/s wr, 466 op/s; 279/1755 objects degraded (15.897%) 2024-04-22T15:23:19.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:19 smithi186 ceph-mon[49804]: pgmap v286: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.9 MiB/s wr, 466 op/s; 279/1755 objects degraded (15.897%) 2024-04-22T15:23:20.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 289480 10.14 MB/sec execute 569 sec latency 80.329 ms 2024-04-22T15:23:20.956 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:20 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:20.956 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:20 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:21.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:20 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:21.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:20 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:21.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 289745 10.13 MB/sec execute 570 sec latency 31.863 ms 2024-04-22T15:23:21.929 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:21 smithi186 ceph-mon[49804]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 406 op/s; 269/1689 objects degraded (15.927%) 2024-04-22T15:23:21.980 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:23:21.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:21 smithi119 ceph-mon[63156]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 406 op/s; 269/1689 objects degraded (15.927%) 2024-04-22T15:23:22.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 290113 10.14 MB/sec execute 571 sec latency 40.134 ms 2024-04-22T15:23:23.001 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:22 smithi119 ceph-mon[63156]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:23.001 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:22 smithi119 ceph-mon[63156]: from='client.44359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:23.002 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:22 smithi119 ceph-mon[63156]: pgmap v288: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 405 op/s; 261/1683 objects degraded (15.508%) 2024-04-22T15:23:23.002 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:22 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 261/1683 objects degraded (15.508%), 35 pgs degraded (PG_DEGRADED) 2024-04-22T15:23:23.013 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:23:23.013 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (8m) 2m ago 18m 15.0M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:23:23.013 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (5m) 2m ago 17m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:23:23.013 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (5m) 4s ago 17m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (8m) 2m ago 17m 47.8M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (12m) 2m ago 12m 38.4M - 17.2.0 e1d6a67b021e 293e5989d82b 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (12m) 4s ago 12m 15.2M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (8m) 2m ago 19m 509M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (7m) 4s ago 17m 453M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (7m) 2m ago 19m 52.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (6m) 4s ago 17m 46.9M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (11m) 2m ago 11m 285M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (11m) 4s ago 11m 85.0M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (8m) 2m ago 17m 10.4M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (8m) 4s ago 17m 20.1M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (4m) 2m ago 16m 357M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (4m) 2m ago 15m 286M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:23:23.014 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (3m) 2m ago 15m 220M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:23:23.015 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (2m) 2m ago 15m 12.8M 1936M 19.0.0-3223-g5883d494 3d983c26e278 2378efa46eca 2024-04-22T15:23:23.015 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (2m) 4s ago 14m 340M 2704M 19.0.0-3223-g5883d494 3d983c26e278 9518e2eb7665 2024-04-22T15:23:23.015 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (83s) 4s ago 14m 292M 2704M 19.0.0-3223-g5883d494 3d983c26e278 e1c8f1367a89 2024-04-22T15:23:23.015 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (45s) 4s ago 13m 271M 2704M 19.0.0-3223-g5883d494 3d983c26e278 31e40cdb753c 2024-04-22T15:23:23.015 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (9s) 4s ago 13m 12.6M 2704M 19.0.0-3223-g5883d494 3d983c26e278 2f6a10f5d68a 2024-04-22T15:23:23.015 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (7m) 2m ago 16m 68.0M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:23:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:22 smithi186 ceph-mon[49804]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:22 smithi186 ceph-mon[49804]: from='client.44359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:22 smithi186 ceph-mon[49804]: pgmap v288: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 405 op/s; 261/1683 objects degraded (15.508%) 2024-04-22T15:23:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:22 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 261/1683 objects degraded (15.508%), 35 pgs degraded (PG_DEGRADED) 2024-04-22T15:23:23.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 290443 10.14 MB/sec execute 572 sec latency 56.027 ms 2024-04-22T15:23:23.669 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:23:23.669 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:23:23.669 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:23:23.669 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:23:23.669 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 7 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 11 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:23:23.670 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:23:24.040 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:23 smithi119 ceph-mon[63156]: from='client.34454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:24.041 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:23 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/3048694892' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:23:24.228 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:23:24.228 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:23:24.228 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:23:24.228 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:23:24.228 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:23:24.228 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:23:24.228 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:23:24.228 INFO:teuthology.orchestra.run.smithi119.stdout: "osd", 2024-04-22T15:23:24.228 INFO:teuthology.orchestra.run.smithi119.stdout: "crash" 2024-04-22T15:23:24.229 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:23:24.229 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "14/23 daemons upgraded", 2024-04-22T15:23:24.229 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:23:24.229 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:23:24.229 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:23:24.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:23 smithi186 ceph-mon[49804]: from='client.34454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:24.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:23 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/3048694892' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:23:24.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 290849 10.15 MB/sec execute 573 sec latency 42.479 ms 2024-04-22T15:23:24.964 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 261/1683 objects degraded (15.508%), 35 pgs degraded 2024-04-22T15:23:24.964 INFO:teuthology.orchestra.run.smithi119.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-22T15:23:24.964 INFO:teuthology.orchestra.run.smithi119.stdout: osd.7 (root=default,host=smithi186) is down 2024-04-22T15:23:24.964 INFO:teuthology.orchestra.run.smithi119.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-04-22T15:23:24.964 INFO:teuthology.orchestra.run.smithi119.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-04-22T15:23:24.964 INFO:teuthology.orchestra.run.smithi119.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 261/1683 objects degraded (15.508%), 35 pgs degraded 2024-04-22T15:23:24.964 INFO:teuthology.orchestra.run.smithi119.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-04-22T15:23:24.964 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-04-22T15:23:24.964 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-04-22T15:23:24.964 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-04-22T15:23:24.964 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-04-22T15:23:24.965 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.7 is active+undersized+degraded, acting [3,0] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.8 is active+undersized+degraded, acting [3,1] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.f is active+undersized+degraded, acting [4,0] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-04-22T15:23:24.966 INFO:teuthology.orchestra.run.smithi119.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-04-22T15:23:24.967 INFO:teuthology.orchestra.run.smithi119.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-04-22T15:23:24.967 INFO:teuthology.orchestra.run.smithi119.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-04-22T15:23:24.967 INFO:teuthology.orchestra.run.smithi119.stdout: pg 4.14 is active+undersized+degraded, acting [3,1] 2024-04-22T15:23:25.308 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:25.308 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:25.308 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:25.308 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:23:25.308 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:23:25.308 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:25.308 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:25.308 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:23:25.308 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:23:25.308 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: Upgrade: Setting container_image for all osd 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-22T15:23:25.309 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-22T15:23:25.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-22T15:23:25.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-22T15:23:25.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-22T15:23:25.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-22T15:23:25.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-22T15:23:25.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: Upgrade: Setting require_osd_release to 19 squid 2024-04-22T15:23:25.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-22T15:23:25.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:25.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:25.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.1 MiB/s wr, 582 op/s; 260/1689 objects degraded (15.394%) 2024-04-22T15:23:25.310 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:25 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/1321859228' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:23:25.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:25.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:25.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:25.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: Upgrade: Setting container_image for all osd 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-22T15:23:25.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: Upgrade: Setting require_osd_release to 19 squid 2024-04-22T15:23:25.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-22T15:23:25.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:25.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:25.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.1 MiB/s wr, 582 op/s; 260/1689 objects degraded (15.394%) 2024-04-22T15:23:25.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/1321859228' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:23:25.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 291321 10.15 MB/sec execute 574 sec latency 32.229 ms 2024-04-22T15:23:26.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:25 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:23:25.607+0000 7f6d6064e7c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-22T15:23:26.324 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:26 smithi119 ceph-mon[63156]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-22T15:23:26.324 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:26 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-22T15:23:26.324 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:26 smithi119 ceph-mon[63156]: osdmap e149: 8 total, 7 up, 8 in 2024-04-22T15:23:26.324 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:26 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi119.ncbqaf"]}]: dispatch 2024-04-22T15:23:26.325 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:26 smithi119 ceph-mon[63156]: Upgrade: It appears safe to stop mds.foofs.smithi119.ncbqaf 2024-04-22T15:23:26.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:26 smithi186 ceph-mon[49804]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-22T15:23:26.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:26 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-22T15:23:26.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:26 smithi186 ceph-mon[49804]: osdmap e149: 8 total, 7 up, 8 in 2024-04-22T15:23:26.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:26 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi119.ncbqaf"]}]: dispatch 2024-04-22T15:23:26.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:26 smithi186 ceph-mon[49804]: Upgrade: It appears safe to stop mds.foofs.smithi119.ncbqaf 2024-04-22T15:23:26.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 291676 10.14 MB/sec execute 575 sec latency 40.118 ms 2024-04-22T15:23:27.379 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:27 smithi119 ceph-mon[63156]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.3 MiB/s wr, 582 op/s; 268/1719 objects degraded (15.590%) 2024-04-22T15:23:27.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:27 smithi186 ceph-mon[49804]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.3 MiB/s wr, 582 op/s; 268/1719 objects degraded (15.590%) 2024-04-22T15:23:27.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 291906 10.13 MB/sec execute 576 sec latency 61.997 ms 2024-04-22T15:23:28.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:27 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:23:27.571+0000 7f6d6064e7c0 -1 osd.7 146 log_to_monitors true 2024-04-22T15:23:28.357 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:28 smithi119 ceph-mon[63156]: from='osd.7 [v2:172.21.15.186:6824/1682807464,v1:172.21.15.186:6825/1682807464]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:23:28.357 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:28 smithi119 ceph-mon[63156]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:23:28.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:28 smithi186 ceph-mon[49804]: from='osd.7 [v2:172.21.15.186:6824/1682807464,v1:172.21.15.186:6825/1682807464]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:23:28.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:28 smithi186 ceph-mon[49804]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:23:28.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 292256 10.14 MB/sec execute 577 sec latency 210.909 ms 2024-04-22T15:23:29.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:23:28 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:23:28.741+0000 7f6d4da9e700 -1 osd.7 146 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:23:29.364 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:29 smithi119 ceph-mon[63156]: Upgrade: Updating mds.foofs.smithi119.ncbqaf 2024-04-22T15:23:29.365 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:29 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:29.365 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:29 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi119.ncbqaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T15:23:29.365 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:29 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:23:29.365 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:29 smithi119 ceph-mon[63156]: Deploying daemon mds.foofs.smithi119.ncbqaf on smithi119 2024-04-22T15:23:29.365 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:29 smithi119 ceph-mon[63156]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T15:23:29.365 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:29 smithi119 ceph-mon[63156]: from='osd.7 [v2:172.21.15.186:6824/1682807464,v1:172.21.15.186:6825/1682807464]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:23:29.365 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:29 smithi119 ceph-mon[63156]: osdmap e150: 8 total, 7 up, 8 in 2024-04-22T15:23:29.365 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:29 smithi119 ceph-mon[63156]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:23:29.365 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:29 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 268/1719 objects degraded (15.590%), 35 pgs degraded (PG_DEGRADED) 2024-04-22T15:23:29.365 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:29 smithi119 ceph-mon[63156]: from='osd.7 ' entity='osd.7' 2024-04-22T15:23:29.365 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:29 smithi119 ceph-mon[63156]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 356 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 618 op/s; 264/1710 objects degraded (15.439%) 2024-04-22T15:23:29.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:29 smithi186 ceph-mon[49804]: Upgrade: Updating mds.foofs.smithi119.ncbqaf 2024-04-22T15:23:29.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:29 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:29.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:29 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi119.ncbqaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T15:23:29.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:29 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:23:29.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:29 smithi186 ceph-mon[49804]: Deploying daemon mds.foofs.smithi119.ncbqaf on smithi119 2024-04-22T15:23:29.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:29 smithi186 ceph-mon[49804]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T15:23:29.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:29 smithi186 ceph-mon[49804]: from='osd.7 [v2:172.21.15.186:6824/1682807464,v1:172.21.15.186:6825/1682807464]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:23:29.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:29 smithi186 ceph-mon[49804]: osdmap e150: 8 total, 7 up, 8 in 2024-04-22T15:23:29.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:29 smithi186 ceph-mon[49804]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-04-22T15:23:29.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:29 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 268/1719 objects degraded (15.590%), 35 pgs degraded (PG_DEGRADED) 2024-04-22T15:23:29.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:29 smithi186 ceph-mon[49804]: from='osd.7 ' entity='osd.7' 2024-04-22T15:23:29.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:29 smithi186 ceph-mon[49804]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 356 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 618 op/s; 264/1710 objects degraded (15.439%) 2024-04-22T15:23:29.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 292638 10.14 MB/sec execute 578 sec latency 135.288 ms 2024-04-22T15:23:30.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:30 smithi186 ceph-mon[49804]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:23:30.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:30 smithi186 ceph-mon[49804]: osd.7 [v2:172.21.15.186:6824/1682807464,v1:172.21.15.186:6825/1682807464] boot 2024-04-22T15:23:30.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:30 smithi186 ceph-mon[49804]: osdmap e151: 8 total, 8 up, 8 in 2024-04-22T15:23:30.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:30 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:23:30.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 292704 10.12 MB/sec execute 579 sec latency 806.379 ms 2024-04-22T15:23:30.634 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:30 smithi119 ceph-mon[63156]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:23:30.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:30 smithi119 ceph-mon[63156]: osd.7 [v2:172.21.15.186:6824/1682807464,v1:172.21.15.186:6825/1682807464] boot 2024-04-22T15:23:30.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:30 smithi119 ceph-mon[63156]: osdmap e151: 8 total, 8 up, 8 in 2024-04-22T15:23:30.635 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:30 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:23:31.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:30 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119[63152]: 2024-04-22T15:23:30.937+0000 7f6b196a5700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T15:23:31.241 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[71857]: 2024-04-22T15:23:31.024+0000 7f426e02a700 -1 bdev(0x561e64350000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 5 2024-04-22T15:23:31.241 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[71857]: 2024-04-22T15:23:31.030+0000 7f426d829700 -1 bdev(0x561e64350000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 7 2024-04-22T15:23:31.242 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[71857]: 2024-04-22T15:23:31.030+0000 7f4269020700 -1 bdev(0x561e64350000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 8 2024-04-22T15:23:31.525 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:23:31 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:23:31.028+0000 7f016ce59700 -1 bdev(0x55f9d8788000 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 3 2024-04-22T15:23:31.534 INFO:teuthology.orchestra.run.smithi119.stdout: 5 292704 10.10 MB/sec execute 580 sec latency 1806.434 ms 2024-04-22T15:23:31.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-mon[63156]: osdmap e152: 8 total, 8 up, 8 in 2024-04-22T15:23:31.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-mon[63156]: pgmap v296: 97 pgs: 1 peering, 7 active+undersized, 34 active+undersized+degraded, 55 active+clean; 350 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.3 MiB/s wr, 527 op/s; 252/1662 objects degraded (15.162%) 2024-04-22T15:23:31.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-mon[63156]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-22T15:23:31.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-mon[63156]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T15:23:31.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-mon[63156]: osdmap e153: 8 total, 8 up, 8 in 2024-04-22T15:23:31.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-mon[63156]: Standby daemon mds.foofs.smithi186.dglszf assigned to filesystem foofs as rank 0 2024-04-22T15:23:31.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-mon[63156]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-22T15:23:31.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-mon[63156]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-22T15:23:31.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-mon[63156]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-22T15:23:31.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:31 smithi119 ceph-mon[63156]: fsmap foofs:1/1 {0=foofs.smithi186.dglszf=up:replay} 2024-04-22T15:23:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:31 smithi186 ceph-mon[49804]: osdmap e152: 8 total, 8 up, 8 in 2024-04-22T15:23:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:31 smithi186 ceph-mon[49804]: pgmap v296: 97 pgs: 1 peering, 7 active+undersized, 34 active+undersized+degraded, 55 active+clean; 350 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.3 MiB/s wr, 527 op/s; 252/1662 objects degraded (15.162%) 2024-04-22T15:23:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:31 smithi186 ceph-mon[49804]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-22T15:23:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:31 smithi186 ceph-mon[49804]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T15:23:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:31 smithi186 ceph-mon[49804]: osdmap e153: 8 total, 8 up, 8 in 2024-04-22T15:23:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:31 smithi186 ceph-mon[49804]: Standby daemon mds.foofs.smithi186.dglszf assigned to filesystem foofs as rank 0 2024-04-22T15:23:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:31 smithi186 ceph-mon[49804]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-22T15:23:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:31 smithi186 ceph-mon[49804]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-22T15:23:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:31 smithi186 ceph-mon[49804]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-22T15:23:32.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:31 smithi186 ceph-mon[49804]: fsmap foofs:1/1 {0=foofs.smithi186.dglszf=up:replay} 2024-04-22T15:23:32.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 292704 10.09 MB/sec execute 581 sec latency 2806.491 ms 2024-04-22T15:23:33.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:32 smithi119 ceph-mon[63156]: osdmap e154: 8 total, 8 up, 8 in 2024-04-22T15:23:33.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:32 smithi119 ceph-mon[63156]: pgmap v299: 97 pgs: 2 remapped+peering, 1 peering, 5 active+undersized, 30 active+undersized+degraded, 59 active+clean; 350 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s wr, 111 op/s; 226/1644 objects degraded (13.747%); 0 B/s, 0 objects/s recovering 2024-04-22T15:23:33.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:32 smithi186 ceph-mon[49804]: osdmap e154: 8 total, 8 up, 8 in 2024-04-22T15:23:33.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:32 smithi186 ceph-mon[49804]: pgmap v299: 97 pgs: 2 remapped+peering, 1 peering, 5 active+undersized, 30 active+undersized+degraded, 59 active+clean; 350 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s wr, 111 op/s; 226/1644 objects degraded (13.747%); 0 B/s, 0 objects/s recovering 2024-04-22T15:23:33.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 292704 10.07 MB/sec execute 582 sec latency 3806.545 ms 2024-04-22T15:23:34.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:33 smithi119 ceph-mon[63156]: reconnect by client.24341 172.21.15.186:0/2852469776 after 0.000999984 2024-04-22T15:23:34.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:33 smithi119 ceph-mon[63156]: osdmap e155: 8 total, 8 up, 8 in 2024-04-22T15:23:34.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:33 smithi119 ceph-mon[63156]: mds.? [v2:172.21.15.186:6832/2825961187,v1:172.21.15.186:6833/2825961187] up:reconnect 2024-04-22T15:23:34.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:33 smithi119 ceph-mon[63156]: fsmap foofs:1/1 {0=foofs.smithi186.dglszf=up:reconnect} 2024-04-22T15:23:34.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:33 smithi119 ceph-mon[63156]: reconnect by client.14550 172.21.15.119:0/2959092703 after 0.0159997 2024-04-22T15:23:34.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:33 smithi119 ceph-mon[63156]: Health check update: Degraded data redundancy: 226/1644 objects degraded (13.747%), 30 pgs degraded (PG_DEGRADED) 2024-04-22T15:23:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:33 smithi186 ceph-mon[49804]: reconnect by client.24341 172.21.15.186:0/2852469776 after 0.000999984 2024-04-22T15:23:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:33 smithi186 ceph-mon[49804]: osdmap e155: 8 total, 8 up, 8 in 2024-04-22T15:23:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:33 smithi186 ceph-mon[49804]: mds.? [v2:172.21.15.186:6832/2825961187,v1:172.21.15.186:6833/2825961187] up:reconnect 2024-04-22T15:23:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:33 smithi186 ceph-mon[49804]: fsmap foofs:1/1 {0=foofs.smithi186.dglszf=up:reconnect} 2024-04-22T15:23:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:33 smithi186 ceph-mon[49804]: reconnect by client.14550 172.21.15.119:0/2959092703 after 0.0159997 2024-04-22T15:23:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:33 smithi186 ceph-mon[49804]: Health check update: Degraded data redundancy: 226/1644 objects degraded (13.747%), 30 pgs degraded (PG_DEGRADED) 2024-04-22T15:23:34.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 292704 10.05 MB/sec execute 583 sec latency 4806.603 ms 2024-04-22T15:23:35.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:34 smithi119 ceph-mon[63156]: mds.? [v2:172.21.15.186:6832/2825961187,v1:172.21.15.186:6833/2825961187] up:rejoin 2024-04-22T15:23:35.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:34 smithi119 ceph-mon[63156]: fsmap foofs:1/1 {0=foofs.smithi186.dglszf=up:rejoin} 2024-04-22T15:23:35.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:34 smithi119 ceph-mon[63156]: pgmap v301: 97 pgs: 2 remapped+peering, 5 peering, 90 active+clean; 339 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 1.7 MiB/s wr, 245 op/s; 4.8 MiB/s, 72 objects/s recovering 2024-04-22T15:23:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:34 smithi186 ceph-mon[49804]: mds.? [v2:172.21.15.186:6832/2825961187,v1:172.21.15.186:6833/2825961187] up:rejoin 2024-04-22T15:23:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:34 smithi186 ceph-mon[49804]: fsmap foofs:1/1 {0=foofs.smithi186.dglszf=up:rejoin} 2024-04-22T15:23:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:34 smithi186 ceph-mon[49804]: pgmap v301: 97 pgs: 2 remapped+peering, 5 peering, 90 active+clean; 339 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 1.7 MiB/s wr, 245 op/s; 4.8 MiB/s, 72 objects/s recovering 2024-04-22T15:23:35.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 292704 10.04 MB/sec execute 584 sec latency 5806.629 ms 2024-04-22T15:23:36.249 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:35 smithi119 ceph-mon[63156]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 226/1644 objects degraded (13.747%), 30 pgs degraded) 2024-04-22T15:23:36.249 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:35 smithi119 ceph-mon[63156]: mds.? [v2:172.21.15.186:6832/2825961187,v1:172.21.15.186:6833/2825961187] up:clientreplay 2024-04-22T15:23:36.250 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:35 smithi119 ceph-mon[63156]: fsmap foofs:1/1 {0=foofs.smithi186.dglszf=up:clientreplay} 2024-04-22T15:23:36.250 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:35 smithi119 ceph-mon[63156]: daemon mds.foofs.smithi186.dglszf is now active in filesystem foofs as rank 0 2024-04-22T15:23:36.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:35 smithi186 ceph-mon[49804]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 226/1644 objects degraded (13.747%), 30 pgs degraded) 2024-04-22T15:23:36.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:35 smithi186 ceph-mon[49804]: mds.? [v2:172.21.15.186:6832/2825961187,v1:172.21.15.186:6833/2825961187] up:clientreplay 2024-04-22T15:23:36.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:35 smithi186 ceph-mon[49804]: fsmap foofs:1/1 {0=foofs.smithi186.dglszf=up:clientreplay} 2024-04-22T15:23:36.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:35 smithi186 ceph-mon[49804]: daemon mds.foofs.smithi186.dglszf is now active in filesystem foofs as rank 0 2024-04-22T15:23:36.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 292845 10.02 MB/sec execute 585 sec latency 6327.922 ms 2024-04-22T15:23:37.024 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:36 smithi119 ceph-mon[63156]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-22T15:23:37.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:36 smithi119 ceph-mon[63156]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-22T15:23:37.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:36 smithi119 ceph-mon[63156]: Cluster is now healthy 2024-04-22T15:23:37.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:36 smithi119 ceph-mon[63156]: mds.? [v2:172.21.15.119:6834/1590565009,v1:172.21.15.119:6835/1590565009] up:boot 2024-04-22T15:23:37.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:36 smithi119 ceph-mon[63156]: mds.? [v2:172.21.15.186:6832/2825961187,v1:172.21.15.186:6833/2825961187] up:active 2024-04-22T15:23:37.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:36 smithi119 ceph-mon[63156]: fsmap foofs:1 {0=foofs.smithi186.dglszf=up:active} 1 up:standby 2024-04-22T15:23:37.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:23:37.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:36 smithi119 ceph-mon[63156]: pgmap v302: 97 pgs: 4 peering, 93 active+clean; 339 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 1.2 MiB/s wr, 173 op/s; 3.9 MiB/s, 53 objects/s recovering 2024-04-22T15:23:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:36 smithi186 ceph-mon[49804]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-22T15:23:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:36 smithi186 ceph-mon[49804]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-22T15:23:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:36 smithi186 ceph-mon[49804]: Cluster is now healthy 2024-04-22T15:23:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:36 smithi186 ceph-mon[49804]: mds.? [v2:172.21.15.119:6834/1590565009,v1:172.21.15.119:6835/1590565009] up:boot 2024-04-22T15:23:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:36 smithi186 ceph-mon[49804]: mds.? [v2:172.21.15.186:6832/2825961187,v1:172.21.15.186:6833/2825961187] up:active 2024-04-22T15:23:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:36 smithi186 ceph-mon[49804]: fsmap foofs:1 {0=foofs.smithi186.dglszf=up:active} 1 up:standby 2024-04-22T15:23:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:23:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:36 smithi186 ceph-mon[49804]: pgmap v302: 97 pgs: 4 peering, 93 active+clean; 339 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 1.2 MiB/s wr, 173 op/s; 3.9 MiB/s, 53 objects/s recovering 2024-04-22T15:23:37.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 293110 10.01 MB/sec execute 586 sec latency 80.492 ms 2024-04-22T15:23:38.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 293265 10.00 MB/sec execute 587 sec latency 61.392 ms 2024-04-22T15:23:39.063 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:38 smithi119 ceph-mon[63156]: pgmap v303: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.4 MiB/s wr, 198 op/s; 3.0 MiB/s, 40 objects/s recovering 2024-04-22T15:23:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:38 smithi186 ceph-mon[49804]: pgmap v303: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.4 MiB/s wr, 198 op/s; 3.0 MiB/s, 40 objects/s recovering 2024-04-22T15:23:39.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 293487 10.00 MB/sec execute 588 sec latency 52.069 ms 2024-04-22T15:23:40.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 293768 10.00 MB/sec execute 589 sec latency 71.647 ms 2024-04-22T15:23:40.893 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:40 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:40.893 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:40 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:41.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:40 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:41.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:40 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:41.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 293985 9.99 MB/sec execute 590 sec latency 72.289 ms 2024-04-22T15:23:41.950 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:41 smithi119 ceph-mon[63156]: pgmap v304: 97 pgs: 97 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.5 MiB/s wr, 192 op/s; 2.7 MiB/s, 35 objects/s recovering 2024-04-22T15:23:42.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:41 smithi186 ceph-mon[49804]: pgmap v304: 97 pgs: 97 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.5 MiB/s wr, 192 op/s; 2.7 MiB/s, 35 objects/s recovering 2024-04-22T15:23:42.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 294216 9.99 MB/sec execute 591 sec latency 104.669 ms 2024-04-22T15:23:42.958 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:42 smithi119 ceph-mon[63156]: pgmap v305: 97 pgs: 97 active+clean; 343 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.4 MiB/s wr, 178 op/s; 2.4 MiB/s, 31 objects/s recovering 2024-04-22T15:23:43.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:42 smithi186 ceph-mon[49804]: pgmap v305: 97 pgs: 97 active+clean; 343 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.4 MiB/s wr, 178 op/s; 2.4 MiB/s, 31 objects/s recovering 2024-04-22T15:23:43.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 294472 9.99 MB/sec execute 592 sec latency 65.409 ms 2024-04-22T15:23:43.968 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:43 smithi119 ceph-mon[63156]: daemon mds.foofs.smithi119.ncbqaf restarted 2024-04-22T15:23:43.969 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:43 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:43.969 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:43 smithi119 ceph-mon[63156]: osdmap e156: 8 total, 8 up, 8 in 2024-04-22T15:23:43.969 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:43 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:43.969 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:43 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:23:44.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:43 smithi186 ceph-mon[49804]: daemon mds.foofs.smithi119.ncbqaf restarted 2024-04-22T15:23:44.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:43 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:44.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:43 smithi186 ceph-mon[49804]: osdmap e156: 8 total, 8 up, 8 in 2024-04-22T15:23:44.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:43 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:44.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:43 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:23:44.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 294761 9.98 MB/sec execute 593 sec latency 53.158 ms 2024-04-22T15:23:44.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:44 smithi119 ceph-mon[63156]: mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210] up:boot 2024-04-22T15:23:44.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:44 smithi119 ceph-mon[63156]: fsmap foofs:1 {0=foofs.smithi186.dglszf=up:active} 1 up:standby 2024-04-22T15:23:44.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:44 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:23:44.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:44 smithi119 ceph-mon[63156]: pgmap v307: 97 pgs: 97 active+clean; 345 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.8 MiB/s wr, 270 op/s; 429 KiB/s, 2 objects/s recovering 2024-04-22T15:23:45.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:44 smithi186 ceph-mon[49804]: mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210] up:boot 2024-04-22T15:23:45.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:44 smithi186 ceph-mon[49804]: fsmap foofs:1 {0=foofs.smithi186.dglszf=up:active} 1 up:standby 2024-04-22T15:23:45.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:44 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:23:45.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:44 smithi186 ceph-mon[49804]: pgmap v307: 97 pgs: 97 active+clean; 345 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.8 MiB/s wr, 270 op/s; 429 KiB/s, 2 objects/s recovering 2024-04-22T15:23:45.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 295131 9.97 MB/sec execute 594 sec latency 43.842 ms 2024-04-22T15:23:46.017 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:45 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:46.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:45 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:46.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 295432 9.97 MB/sec execute 595 sec latency 35.857 ms 2024-04-22T15:23:47.037 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:46 smithi119 ceph-mon[63156]: pgmap v308: 97 pgs: 97 active+clean; 345 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 2.9 MiB/s wr, 278 op/s 2024-04-22T15:23:47.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:46 smithi186 ceph-mon[49804]: pgmap v308: 97 pgs: 97 active+clean; 345 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 2.9 MiB/s wr, 278 op/s 2024-04-22T15:23:47.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 295802 9.98 MB/sec execute 596 sec latency 45.968 ms 2024-04-22T15:23:48.535 INFO:teuthology.orchestra.run.smithi119.stdout: 5 296244 9.98 MB/sec execute 597 sec latency 54.068 ms 2024-04-22T15:23:49.066 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:48 smithi119 ceph-mon[63156]: pgmap v309: 97 pgs: 97 active+clean; 349 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 4.0 MiB/s wr, 424 op/s 2024-04-22T15:23:49.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:48 smithi186 ceph-mon[49804]: pgmap v309: 97 pgs: 97 active+clean; 349 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 4.0 MiB/s wr, 424 op/s 2024-04-22T15:23:49.537 INFO:teuthology.orchestra.run.smithi119.stdout: 5 296649 9.97 MB/sec execute 598 sec latency 52.065 ms 2024-04-22T15:23:50.537 INFO:teuthology.orchestra.run.smithi119.stdout: 5 296985 9.98 MB/sec execute 599 sec latency 36.446 ms 2024-04-22T15:23:51.102 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:50 smithi119 ceph-mon[63156]: pgmap v310: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 MiB/s wr, 434 op/s 2024-04-22T15:23:51.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:50 smithi186 ceph-mon[49804]: pgmap v310: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 MiB/s wr, 434 op/s 2024-04-22T15:23:51.537 INFO:teuthology.orchestra.run.smithi119.stdout: 5 cleanup 600 sec 2024-04-22T15:23:51.747 INFO:teuthology.orchestra.run.smithi119.stdout: 0 cleanup 600 sec 2024-04-22T15:23:51.747 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:23:51.747 INFO:teuthology.orchestra.run.smithi119.stdout: Operation Count AvgLat MaxLat 2024-04-22T15:23:51.747 INFO:teuthology.orchestra.run.smithi119.stdout: ---------------------------------------- 2024-04-22T15:23:51.747 INFO:teuthology.orchestra.run.smithi119.stdout: NTCreateX 192292 4.442 2053.856 2024-04-22T15:23:51.747 INFO:teuthology.orchestra.run.smithi119.stdout: Close 141177 0.135 18.798 2024-04-22T15:23:51.747 INFO:teuthology.orchestra.run.smithi119.stdout: Rename 8147 10.076 1091.008 2024-04-22T15:23:51.747 INFO:teuthology.orchestra.run.smithi119.stdout: Unlink 38892 6.910 6306.485 2024-04-22T15:23:51.747 INFO:teuthology.orchestra.run.smithi119.stdout: Qpathinfo 174497 1.133 1067.991 2024-04-22T15:23:51.748 INFO:teuthology.orchestra.run.smithi119.stdout: Qfileinfo 30423 0.007 2.617 2024-04-22T15:23:51.748 INFO:teuthology.orchestra.run.smithi119.stdout: Qfsinfo 31930 0.616 21.155 2024-04-22T15:23:51.748 INFO:teuthology.orchestra.run.smithi119.stdout: Sfileinfo 15704 12.410 1070.852 2024-04-22T15:23:51.748 INFO:teuthology.orchestra.run.smithi119.stdout: Find 67361 0.956 48.847 2024-04-22T15:23:51.748 INFO:teuthology.orchestra.run.smithi119.stdout: WriteX 94999 13.602 6327.915 2024-04-22T15:23:51.748 INFO:teuthology.orchestra.run.smithi119.stdout: ReadX 301533 0.010 2.803 2024-04-22T15:23:51.748 INFO:teuthology.orchestra.run.smithi119.stdout: LockX 628 0.387 2.867 2024-04-22T15:23:51.748 INFO:teuthology.orchestra.run.smithi119.stdout: UnlockX 628 0.346 5.942 2024-04-22T15:23:51.748 INFO:teuthology.orchestra.run.smithi119.stdout: Flush 13492 0.002 0.029 2024-04-22T15:23:51.748 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:23:51.748 INFO:teuthology.orchestra.run.smithi119.stdout:Throughput 9.97591 MB/sec 5 clients 5 procs max_latency=6327.922 ms 2024-04-22T15:23:51.749 DEBUG:teuthology.orchestra.run.smithi119:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-04-22T15:23:51.786 DEBUG:teuthology.orchestra.run.smithi119:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-04-22T15:23:51.914 DEBUG:teuthology.orchestra.run.smithi119:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-04-22T15:23:51.939 INFO:teuthology.orchestra.run.smithi119.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-22T15:23:51.939 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:23:51.939 INFO:teuthology.orchestra.run.smithi119.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-04-22T15:23:51.940 INFO:teuthology.orchestra.run.smithi119.stdout:4 of 5 processes prepared for launch 0 sec 2024-04-22T15:23:52.040 INFO:teuthology.orchestra.run.smithi119.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-22T15:23:52.040 INFO:teuthology.orchestra.run.smithi119.stdout:releasing clients 2024-04-22T15:23:52.874 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:52 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:52.874 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:52 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:53.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:52 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:53.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:52 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:53.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:53 smithi119 ceph-mon[63156]: pgmap v311: 97 pgs: 97 active+clean; 349 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.3 MiB/s wr, 440 op/s 2024-04-22T15:23:53.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:53 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:53.906 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:53 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:54.041 INFO:teuthology.orchestra.run.smithi119.stdout: 5 384 24.95 MB/sec execute 1 sec latency 55.070 ms 2024-04-22T15:23:54.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:53 smithi186 ceph-mon[49804]: pgmap v311: 97 pgs: 97 active+clean; 349 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.3 MiB/s wr, 440 op/s 2024-04-22T15:23:54.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:53 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:54.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:53 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:23:54.932 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:54 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:55.041 INFO:teuthology.orchestra.run.smithi119.stdout: 5 554 26.06 MB/sec execute 2 sec latency 51.992 ms 2024-04-22T15:23:55.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:54 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:55.458 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:23:55.945 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:55 smithi119 ceph-mon[63156]: pgmap v312: 97 pgs: 97 active+clean; 401 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.7 MiB/s wr, 476 op/s 2024-04-22T15:23:56.041 INFO:teuthology.orchestra.run.smithi119.stdout: 5 910 21.76 MB/sec execute 3 sec latency 34.542 ms 2024-04-22T15:23:56.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:55 smithi186 ceph-mon[49804]: pgmap v312: 97 pgs: 97 active+clean; 401 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.7 MiB/s wr, 476 op/s 2024-04-22T15:23:56.519 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:23:56.519 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (9m) 4s ago 18m 24.6M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:23:56.519 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (6m) 4s ago 18m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (6m) 38s ago 17m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (9m) 4s ago 18m 48.5M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (13s) 4s ago 12m 14.8M - 19.0.0-3223-g5883d494 3d983c26e278 5019782f86de 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (12m) 38s ago 12m 15.2M - 17.2.0 e1d6a67b021e 392fac392fd9 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (8m) 4s ago 19m 513M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (8m) 38s ago 17m 453M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (7m) 4s ago 19m 61.5M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (7m) 38s ago 17m 46.9M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (12m) 4s ago 12m 284M - 4.0 e1d6a67b021e 65d0fee12613 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (12m) 38s ago 12m 85.0M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (9m) 4s ago 18m 11.2M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (9m) 38s ago 17m 20.1M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (5m) 4s ago 17m 433M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (4m) 4s ago 16m 384M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:23:56.520 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (3m) 4s ago 16m 387M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:23:56.521 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (3m) 4s ago 15m 353M 1936M 19.0.0-3223-g5883d494 3d983c26e278 2378efa46eca 2024-04-22T15:23:56.521 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (2m) 38s ago 15m 340M 2704M 19.0.0-3223-g5883d494 3d983c26e278 9518e2eb7665 2024-04-22T15:23:56.521 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (117s) 38s ago 14m 292M 2704M 19.0.0-3223-g5883d494 3d983c26e278 e1c8f1367a89 2024-04-22T15:23:56.521 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (79s) 38s ago 14m 271M 2704M 19.0.0-3223-g5883d494 3d983c26e278 31e40cdb753c 2024-04-22T15:23:56.521 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (42s) 38s ago 14m 12.6M 2704M 19.0.0-3223-g5883d494 3d983c26e278 2f6a10f5d68a 2024-04-22T15:23:56.521 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (8m) 4s ago 17m 70.4M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:23:56.959 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:56 smithi119 ceph-mon[63156]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:57.041 INFO:teuthology.orchestra.run.smithi119.stdout: 5 1291 18.79 MB/sec execute 4 sec latency 39.220 ms 2024-04-22T15:23:57.129 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:23:57.129 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:23:57.129 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:23:57.130 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:23:57.130 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:23:57.130 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:23:57.130 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:23:57.130 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:23:57.130 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:23:57.130 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:23:57.130 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:23:57.130 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T15:23:57.130 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:23:57.130 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:23:57.130 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:23:57.131 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T15:23:57.131 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 13 2024-04-22T15:23:57.131 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:23:57.131 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:23:57.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:56 smithi186 ceph-mon[49804]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:57.665 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:23:57.666 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:23:57.666 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:23:57.666 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:23:57.666 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:23:57.666 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:23:57.666 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:23:57.666 INFO:teuthology.orchestra.run.smithi119.stdout: "osd", 2024-04-22T15:23:57.666 INFO:teuthology.orchestra.run.smithi119.stdout: "crash" 2024-04-22T15:23:57.666 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:23:57.666 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "15/23 daemons upgraded", 2024-04-22T15:23:57.666 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading mds daemons", 2024-04-22T15:23:57.666 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:23:57.667 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:23:57.977 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:57 smithi119 ceph-mon[63156]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:57.977 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:57 smithi119 ceph-mon[63156]: from='client.34486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:57.977 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:57 smithi119 ceph-mon[63156]: pgmap v313: 97 pgs: 97 active+clean; 400 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 449 op/s 2024-04-22T15:23:57.977 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:57 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/563035626' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:23:58.041 INFO:teuthology.orchestra.run.smithi119.stdout: 5 cleanup 5 sec 2024-04-22T15:23:58.198 INFO:teuthology.orchestra.run.smithi119.stdout:[1837] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000ce4300000013' failed - Device or resource busy 2024-04-22T15:23:58.198 INFO:teuthology.orchestra.run.smithi119.stdout:[1837] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000ce3e0000001c' failed - Device or resource busy 2024-04-22T15:23:58.198 INFO:teuthology.orchestra.run.smithi119.stdout:[1837] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000ce4d0000001d' failed - Device or resource busy 2024-04-22T15:23:58.198 INFO:teuthology.orchestra.run.smithi119.stdout:[1837] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000ce840000001e' failed - Device or resource busy 2024-04-22T15:23:58.199 INFO:teuthology.orchestra.run.smithi119.stdout:[1837] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000ce740000001f' failed - Device or resource busy 2024-04-22T15:23:58.199 INFO:teuthology.orchestra.run.smithi119.stdout:[1837] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000ce6a00000018' failed - Device or resource busy 2024-04-22T15:23:58.199 INFO:teuthology.orchestra.run.smithi119.stdout:[1837] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000ce4700000014' failed - Device or resource busy 2024-04-22T15:23:58.200 INFO:teuthology.orchestra.run.smithi119.stdout:[1837] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000ce890000001a' failed - Device or resource busy 2024-04-22T15:23:58.235 INFO:teuthology.orchestra.run.smithi119.stdout:[1702] unlink './clients/client3/~dmtmp/PARADOX/.nfs000001000000cf5100000019' failed - Device or resource busy 2024-04-22T15:23:58.235 INFO:teuthology.orchestra.run.smithi119.stdout:[1702] unlink './clients/client3/~dmtmp/PARADOX/.nfs000001000000cf4300000027' failed - Device or resource busy 2024-04-22T15:23:58.236 INFO:teuthology.orchestra.run.smithi119.stdout:[1727] unlink './clients/client4/~dmtmp/PARADOX/.nfs000001000000cf4f00000021' failed - Device or resource busy 2024-04-22T15:23:58.236 INFO:teuthology.orchestra.run.smithi119.stdout:[1727] unlink './clients/client4/~dmtmp/PARADOX/.nfs000001000000ce6400000022' failed - Device or resource busy 2024-04-22T15:23:58.236 INFO:teuthology.orchestra.run.smithi119.stdout:[1727] unlink './clients/client4/~dmtmp/PARADOX/.nfs000001000000ce720000002a' failed - Device or resource busy 2024-04-22T15:23:58.237 INFO:teuthology.orchestra.run.smithi119.stdout:[1727] unlink './clients/client4/~dmtmp/PARADOX/.nfs000001000000ce8100000023' failed - Device or resource busy 2024-04-22T15:23:58.237 INFO:teuthology.orchestra.run.smithi119.stdout:[1727] unlink './clients/client4/~dmtmp/PARADOX/.nfs000001000000ce860000002c' failed - Device or resource busy 2024-04-22T15:23:58.238 INFO:teuthology.orchestra.run.smithi119.stdout:[1727] unlink './clients/client4/~dmtmp/PARADOX/.nfs000001000000ce8b00000029' failed - Device or resource busy 2024-04-22T15:23:58.238 INFO:teuthology.orchestra.run.smithi119.stdout:[1837] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000ce6f0000001b' failed - Device or resource busy 2024-04-22T15:23:58.238 INFO:teuthology.orchestra.run.smithi119.stdout:[1727] unlink './clients/client4/~dmtmp/PARADOX/.nfs000001000000ce6d00000026' failed - Device or resource busy 2024-04-22T15:23:58.239 INFO:teuthology.orchestra.run.smithi119.stdout:[1702] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000cf5300000020' failed - Device or resource busy 2024-04-22T15:23:58.239 INFO:teuthology.orchestra.run.smithi119.stdout:[1702] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000cf450000002e' failed - Device or resource busy 2024-04-22T15:23:58.241 INFO:teuthology.orchestra.run.smithi119.stdout:[1727] unlink './clients/client4/~dmtmp/PARADOX/.nfs000001000000cf420000002b' failed - Device or resource busy 2024-04-22T15:23:58.251 INFO:teuthology.orchestra.run.smithi119.stdout:[1727] unlink './clients/client4/~dmtmp/PARADOX/.nfs000001000000ce5a00000025' failed - Device or resource busy 2024-04-22T15:23:58.252 INFO:teuthology.orchestra.run.smithi119.stdout:[1837] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000ce5700000017' failed - Device or resource busy 2024-04-22T15:23:58.253 INFO:teuthology.orchestra.run.smithi119.stdout:[1837] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000ce7f00000016' failed - Device or resource busy 2024-04-22T15:23:58.255 INFO:teuthology.orchestra.run.smithi119.stdout:[1837] unlink './clients/client0/~dmtmp/PARADOX/.nfs000001000000ce6100000015' failed - Device or resource busy 2024-04-22T15:23:58.258 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:23:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:57 smithi186 ceph-mon[49804]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:57 smithi186 ceph-mon[49804]: from='client.34486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:57 smithi186 ceph-mon[49804]: pgmap v313: 97 pgs: 97 active+clean; 400 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 449 op/s 2024-04-22T15:23:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:57 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/563035626' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:23:58.318 INFO:teuthology.orchestra.run.smithi119.stdout: 0 cleanup 5 sec 2024-04-22T15:23:58.318 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:23:58.318 INFO:teuthology.orchestra.run.smithi119.stdout: Operation Count AvgLat MaxLat 2024-04-22T15:23:58.318 INFO:teuthology.orchestra.run.smithi119.stdout: ---------------------------------------- 2024-04-22T15:23:58.318 INFO:teuthology.orchestra.run.smithi119.stdout: NTCreateX 1235 5.722 52.274 2024-04-22T15:23:58.318 INFO:teuthology.orchestra.run.smithi119.stdout: Close 1067 5.210 40.059 2024-04-22T15:23:58.318 INFO:teuthology.orchestra.run.smithi119.stdout: Rename 55 9.728 35.954 2024-04-22T15:23:58.318 INFO:teuthology.orchestra.run.smithi119.stdout: Unlink 114 5.473 26.207 2024-04-22T15:23:58.318 INFO:teuthology.orchestra.run.smithi119.stdout: Qpathinfo 824 1.725 39.780 2024-04-22T15:23:58.318 INFO:teuthology.orchestra.run.smithi119.stdout: Qfileinfo 420 14.168 51.753 2024-04-22T15:23:58.319 INFO:teuthology.orchestra.run.smithi119.stdout: Qfsinfo 165 0.657 6.372 2024-04-22T15:23:58.319 INFO:teuthology.orchestra.run.smithi119.stdout: Sfileinfo 170 10.998 30.384 2024-04-22T15:23:58.319 INFO:teuthology.orchestra.run.smithi119.stdout: Find 317 1.018 6.470 2024-04-22T15:23:58.319 INFO:teuthology.orchestra.run.smithi119.stdout: WriteX 1770 0.024 0.101 2024-04-22T15:23:58.319 INFO:teuthology.orchestra.run.smithi119.stdout: ReadX 1345 0.016 0.480 2024-04-22T15:23:58.319 INFO:teuthology.orchestra.run.smithi119.stdout: Flush 135 11.557 27.905 2024-04-22T15:23:58.319 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:23:58.319 INFO:teuthology.orchestra.run.smithi119.stdout:Throughput 18.7865 MB/sec 5 clients 5 procs max_latency=55.070 ms 2024-04-22T15:23:58.320 DEBUG:teuthology.parallel:result is None 2024-04-22T15:23:58.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:58 smithi119 ceph-mon[63156]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:58.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:58 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/2418518027' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:23:59.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:58 smithi186 ceph-mon[49804]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:23:59.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:58 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/2418518027' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:24:00.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: pgmap v314: 97 pgs: 97 active+clean; 372 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 636 op/s 2024-04-22T15:24:00.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:00.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:00.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:00.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:24:00.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:00.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:00.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:00.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:00.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:00.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:00.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:00.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:23:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi186.dglszf"]}]: dispatch 2024-04-22T15:24:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: pgmap v314: 97 pgs: 97 active+clean; 372 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 636 op/s 2024-04-22T15:24:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:24:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:00.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:00.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:23:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi186.dglszf"]}]: dispatch 2024-04-22T15:24:01.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:00 smithi119 ceph-mon[63156]: Detected new or changed devices on smithi119 2024-04-22T15:24:01.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:00 smithi119 ceph-mon[63156]: Upgrade: It appears safe to stop mds.foofs.smithi186.dglszf 2024-04-22T15:24:01.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:00 smithi119 ceph-mon[63156]: pgmap v315: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 12 MiB/s wr, 477 op/s 2024-04-22T15:24:01.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:00 smithi186 ceph-mon[49804]: Detected new or changed devices on smithi119 2024-04-22T15:24:01.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:00 smithi186 ceph-mon[49804]: Upgrade: It appears safe to stop mds.foofs.smithi186.dglszf 2024-04-22T15:24:01.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:00 smithi186 ceph-mon[49804]: pgmap v315: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 12 MiB/s wr, 477 op/s 2024-04-22T15:24:02.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:02 smithi186 ceph-mon[49804]: Upgrade: Updating mds.foofs.smithi186.dglszf 2024-04-22T15:24:02.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:02 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:02.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:02 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi186.dglszf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T15:24:02.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:02 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:02.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:02 smithi186 ceph-mon[49804]: Deploying daemon mds.foofs.smithi186.dglszf on smithi186 2024-04-22T15:24:02.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:02 smithi119 ceph-mon[63156]: Upgrade: Updating mds.foofs.smithi186.dglszf 2024-04-22T15:24:02.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:02 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:02.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:02 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi186.dglszf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T15:24:02.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:02 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:02.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:02 smithi119 ceph-mon[63156]: Deploying daemon mds.foofs.smithi186.dglszf on smithi186 2024-04-22T15:24:03.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:03 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119[63152]: 2024-04-22T15:24:03.138+0000 7f6b196a5700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T15:24:03.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:03 smithi119 ceph-mon[63156]: pgmap v316: 97 pgs: 97 active+clean; 322 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 12 MiB/s wr, 484 op/s 2024-04-22T15:24:03.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:03 smithi119 ceph-mon[63156]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-22T15:24:03.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:03 smithi119 ceph-mon[63156]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T15:24:03.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:03 smithi119 ceph-mon[63156]: osdmap e157: 8 total, 8 up, 8 in 2024-04-22T15:24:03.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:03 smithi119 ceph-mon[63156]: Standby daemon mds.foofs.smithi119.ncbqaf assigned to filesystem foofs as rank 0 2024-04-22T15:24:03.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:03 smithi119 ceph-mon[63156]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-22T15:24:03.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:03 smithi119 ceph-mon[63156]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-22T15:24:03.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:03 smithi119 ceph-mon[63156]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-22T15:24:03.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:03 smithi119 ceph-mon[63156]: fsmap foofs:1/1 {0=foofs.smithi119.ncbqaf=up:replay} 2024-04-22T15:24:03.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:03 smithi186 ceph-mon[49804]: pgmap v316: 97 pgs: 97 active+clean; 322 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 12 MiB/s wr, 484 op/s 2024-04-22T15:24:03.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:03 smithi186 ceph-mon[49804]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-22T15:24:03.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:03 smithi186 ceph-mon[49804]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T15:24:03.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:03 smithi186 ceph-mon[49804]: osdmap e157: 8 total, 8 up, 8 in 2024-04-22T15:24:03.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:03 smithi186 ceph-mon[49804]: Standby daemon mds.foofs.smithi119.ncbqaf assigned to filesystem foofs as rank 0 2024-04-22T15:24:03.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:03 smithi186 ceph-mon[49804]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-22T15:24:03.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:03 smithi186 ceph-mon[49804]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-22T15:24:03.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:03 smithi186 ceph-mon[49804]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-22T15:24:03.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:03 smithi186 ceph-mon[49804]: fsmap foofs:1/1 {0=foofs.smithi119.ncbqaf=up:replay} 2024-04-22T15:24:05.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:05 smithi119 ceph-mon[63156]: reconnect by client.14550 172.21.15.119:0/2959092703 after 0.000999976 2024-04-22T15:24:05.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:05 smithi119 ceph-mon[63156]: reconnect by client.24341 172.21.15.186:0/2852469776 after 0.000999976 2024-04-22T15:24:05.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:05 smithi119 ceph-mon[63156]: mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210] up:reconnect 2024-04-22T15:24:05.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:05 smithi119 ceph-mon[63156]: fsmap foofs:1/1 {0=foofs.smithi119.ncbqaf=up:reconnect} 2024-04-22T15:24:05.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:05 smithi119 ceph-mon[63156]: pgmap v318: 97 pgs: 97 active+clean; 254 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 7.1 MiB/s wr, 351 op/s 2024-04-22T15:24:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:05 smithi186 ceph-mon[49804]: reconnect by client.14550 172.21.15.119:0/2959092703 after 0.000999976 2024-04-22T15:24:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:05 smithi186 ceph-mon[49804]: reconnect by client.24341 172.21.15.186:0/2852469776 after 0.000999976 2024-04-22T15:24:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:05 smithi186 ceph-mon[49804]: mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210] up:reconnect 2024-04-22T15:24:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:05 smithi186 ceph-mon[49804]: fsmap foofs:1/1 {0=foofs.smithi119.ncbqaf=up:reconnect} 2024-04-22T15:24:05.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:05 smithi186 ceph-mon[49804]: pgmap v318: 97 pgs: 97 active+clean; 254 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 7.1 MiB/s wr, 351 op/s 2024-04-22T15:24:06.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:06 smithi119 ceph-mon[63156]: mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210] up:rejoin 2024-04-22T15:24:06.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:06 smithi119 ceph-mon[63156]: fsmap foofs:1/1 {0=foofs.smithi119.ncbqaf=up:rejoin} 2024-04-22T15:24:06.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:06 smithi119 ceph-mon[63156]: daemon mds.foofs.smithi119.ncbqaf is now active in filesystem foofs as rank 0 2024-04-22T15:24:06.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:24:06.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:06 smithi186 ceph-mon[49804]: mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210] up:rejoin 2024-04-22T15:24:06.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:06 smithi186 ceph-mon[49804]: fsmap foofs:1/1 {0=foofs.smithi119.ncbqaf=up:rejoin} 2024-04-22T15:24:06.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:06 smithi186 ceph-mon[49804]: daemon mds.foofs.smithi119.ncbqaf is now active in filesystem foofs as rank 0 2024-04-22T15:24:06.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:24:07.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:07 smithi119 ceph-mon[63156]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-22T15:24:07.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:07 smithi119 ceph-mon[63156]: mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210] up:active 2024-04-22T15:24:07.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:07 smithi119 ceph-mon[63156]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 2024-04-22T15:24:07.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:07 smithi119 ceph-mon[63156]: pgmap v319: 97 pgs: 97 active+clean; 255 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.4 MiB/s wr, 331 op/s 2024-04-22T15:24:07.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:07 smithi186 ceph-mon[49804]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-22T15:24:07.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:07 smithi186 ceph-mon[49804]: mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210] up:active 2024-04-22T15:24:07.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:07 smithi186 ceph-mon[49804]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 2024-04-22T15:24:07.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:07 smithi186 ceph-mon[49804]: pgmap v319: 97 pgs: 97 active+clean; 255 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.4 MiB/s wr, 331 op/s 2024-04-22T15:24:09.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:08 smithi119 ceph-mon[63156]: pgmap v320: 97 pgs: 97 active+clean; 244 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 1.3 MiB/s wr, 104 op/s 2024-04-22T15:24:09.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:08 smithi186 ceph-mon[49804]: pgmap v320: 97 pgs: 97 active+clean; 244 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 1.3 MiB/s wr, 104 op/s 2024-04-22T15:24:11.050 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:11.051 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:11.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:11.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:11 smithi186 ceph-mon[49804]: pgmap v321: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 511 KiB/s wr, 81 op/s 2024-04-22T15:24:12.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:11 smithi119 ceph-mon[63156]: pgmap v321: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 511 KiB/s wr, 81 op/s 2024-04-22T15:24:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:13 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:13 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:13 smithi186 ceph-mon[49804]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-22T15:24:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:13 smithi186 ceph-mon[49804]: Cluster is now healthy 2024-04-22T15:24:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:13 smithi186 ceph-mon[49804]: mds.? [v2:172.21.15.186:6832/129378873,v1:172.21.15.186:6833/129378873] up:boot 2024-04-22T15:24:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:13 smithi186 ceph-mon[49804]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 1 up:standby 2024-04-22T15:24:14.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:13 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:24:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:13 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:14.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:13 smithi186 ceph-mon[49804]: pgmap v322: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 66 KiB/s wr, 53 op/s 2024-04-22T15:24:14.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:13 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:14.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:13 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:14.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:13 smithi119 ceph-mon[63156]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-22T15:24:14.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:13 smithi119 ceph-mon[63156]: Cluster is now healthy 2024-04-22T15:24:14.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:13 smithi119 ceph-mon[63156]: mds.? [v2:172.21.15.186:6832/129378873,v1:172.21.15.186:6833/129378873] up:boot 2024-04-22T15:24:14.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:13 smithi119 ceph-mon[63156]: fsmap foofs:1 {0=foofs.smithi119.ncbqaf=up:active} 1 up:standby 2024-04-22T15:24:14.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:13 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:24:14.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:13 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:14.243 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:13 smithi119 ceph-mon[63156]: pgmap v322: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 66 KiB/s wr, 53 op/s 2024-04-22T15:24:15.152 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:14 smithi119 ceph-mon[63156]: pgmap v323: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 54 KiB/s wr, 34 op/s 2024-04-22T15:24:15.253 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:14 smithi186 ceph-mon[49804]: pgmap v323: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 54 KiB/s wr, 34 op/s 2024-04-22T15:24:17.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:16 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:17.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:16 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:17.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:16 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:17.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:16 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:18.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:17 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:18.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:17 smithi186 ceph-mon[49804]: pgmap v324: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 50 KiB/s wr, 17 op/s 2024-04-22T15:24:18.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:17 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:18.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:17 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:18.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:17 smithi119 ceph-mon[63156]: pgmap v324: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 50 KiB/s wr, 17 op/s 2024-04-22T15:24:18.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:17 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:19.895 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:19 smithi186 ceph-mon[49804]: pgmap v325: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 14 KiB/s wr, 16 op/s 2024-04-22T15:24:20.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:19 smithi119 ceph-mon[63156]: pgmap v325: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 14 KiB/s wr, 16 op/s 2024-04-22T15:24:21.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: Detected new or changed devices on smithi186 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: Upgrade: Setting container_image for all mds 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi119.ncbqaf"}]': finished 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi186.dglszf"}]': finished 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: Upgrade: Setting container_image for all rgw 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:24:21.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: Upgrade: Setting container_image for all iscsi 2024-04-22T15:24:21.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.528 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:21 smithi186 ceph-mon[49804]: pgmap v326: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 12 KiB/s wr, 8 op/s 2024-04-22T15:24:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: Detected new or changed devices on smithi186 2024-04-22T15:24:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:24:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: Upgrade: Setting container_image for all mds 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi119.ncbqaf"}]: dispatch 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi119.ncbqaf"}]': finished 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi186.dglszf"}]: dispatch 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi186.dglszf"}]': finished 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: Upgrade: Setting container_image for all rgw 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:24:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:24:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: Upgrade: Setting container_image for all iscsi 2024-04-22T15:24:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:21 smithi119 ceph-mon[63156]: pgmap v326: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 12 KiB/s wr, 8 op/s 2024-04-22T15:24:23.362 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: Upgrade: Updating nfs.foo.0.0.smithi119.zcjqol (1/2) 2024-04-22T15:24:23.363 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:23.363 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: Creating key for client.nfs.foo.0.0.smithi119.zcjqol 2024-04-22T15:24:23.363 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi119.zcjqol", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T15:24:23.363 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-22T15:24:23.363 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T15:24:23.363 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T15:24:23.363 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:23.363 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T15:24:23.363 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T15:24:23.363 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: Rados config object exists: conf-nfs.foo 2024-04-22T15:24:23.364 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: Creating key for client.nfs.foo.0.0.smithi119.zcjqol-rgw 2024-04-22T15:24:23.364 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi119.zcjqol-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:24:23.364 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: Bind address in nfs.foo.0.0.smithi119.zcjqol's ganesha conf is defaulting to empty 2024-04-22T15:24:23.364 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:23.364 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: Deploying daemon nfs.foo.0.0.smithi119.zcjqol on smithi119 2024-04-22T15:24:23.364 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:23 smithi119 ceph-mon[63156]: pgmap v327: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 9.8 KiB/s wr, 3 op/s 2024-04-22T15:24:23.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: Upgrade: Updating nfs.foo.0.0.smithi119.zcjqol (1/2) 2024-04-22T15:24:23.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:23.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: Creating key for client.nfs.foo.0.0.smithi119.zcjqol 2024-04-22T15:24:23.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi119.zcjqol", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: Rados config object exists: conf-nfs.foo 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: Creating key for client.nfs.foo.0.0.smithi119.zcjqol-rgw 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi119.zcjqol-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: Bind address in nfs.foo.0.0.smithi119.zcjqol's ganesha conf is defaulting to empty 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: Deploying daemon nfs.foo.0.0.smithi119.zcjqol on smithi119 2024-04-22T15:24:23.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:23 smithi186 ceph-mon[49804]: pgmap v327: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 9.8 KiB/s wr, 3 op/s 2024-04-22T15:24:26.056 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:26.056 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:26.056 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:25 smithi119 ceph-mon[63156]: pgmap v328: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 597 B/s wr, 0 op/s 2024-04-22T15:24:26.162 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:26.162 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:26.162 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:25 smithi186 ceph-mon[49804]: pgmap v328: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 597 B/s wr, 0 op/s 2024-04-22T15:24:27.139 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:26 smithi186 ceph-mon[49804]: Evicting (and blocklisting) client session 14550 (172.21.15.119:0/2959092703) 2024-04-22T15:24:27.140 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:26 smithi186 ceph-mon[49804]: from='mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210]' entity='mds.foofs.smithi119.ncbqaf' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.119:0/2959092703"}]: dispatch 2024-04-22T15:24:27.140 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:26 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:27.140 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:26 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:27.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:26 smithi119 ceph-mon[63156]: Evicting (and blocklisting) client session 14550 (172.21.15.119:0/2959092703) 2024-04-22T15:24:27.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:26 smithi119 ceph-mon[63156]: from='mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210]' entity='mds.foofs.smithi119.ncbqaf' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.119:0/2959092703"}]: dispatch 2024-04-22T15:24:27.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:26 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:27.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:26 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:28.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:27 smithi119 ceph-mon[63156]: from='mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210]' entity='mds.foofs.smithi119.ncbqaf' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.119:0/2959092703"}]': finished 2024-04-22T15:24:28.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:27 smithi119 ceph-mon[63156]: osdmap e158: 8 total, 8 up, 8 in 2024-04-22T15:24:28.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:27 smithi119 ceph-mon[63156]: pgmap v330: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-04-22T15:24:28.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:27 smithi186 ceph-mon[49804]: from='mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210]' entity='mds.foofs.smithi119.ncbqaf' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.119:0/2959092703"}]': finished 2024-04-22T15:24:28.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:27 smithi186 ceph-mon[49804]: osdmap e158: 8 total, 8 up, 8 in 2024-04-22T15:24:28.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:27 smithi186 ceph-mon[49804]: pgmap v330: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-04-22T15:24:28.619 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:24:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: Upgrade: Updating nfs.foo.1.0.smithi186.mklmhy (2/2) 2024-04-22T15:24:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: Creating key for client.nfs.foo.1.0.smithi186.mklmhy 2024-04-22T15:24:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi186.mklmhy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T15:24:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-22T15:24:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T15:24:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T15:24:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T15:24:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T15:24:28.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: Rados config object exists: conf-nfs.foo 2024-04-22T15:24:28.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: Creating key for client.nfs.foo.1.0.smithi186.mklmhy-rgw 2024-04-22T15:24:28.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi186.mklmhy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:24:28.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: Bind address in nfs.foo.1.0.smithi186.mklmhy's ganesha conf is defaulting to empty 2024-04-22T15:24:28.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:28.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: Deploying daemon nfs.foo.1.0.smithi186.mklmhy on smithi186 2024-04-22T15:24:28.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: from='client.34538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:24:28.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:28 smithi119 ceph-mon[63156]: pgmap v331: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 818 B/s wr, 3 op/s 2024-04-22T15:24:29.166 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: Upgrade: Updating nfs.foo.1.0.smithi186.mklmhy (2/2) 2024-04-22T15:24:29.166 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:29.166 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: Creating key for client.nfs.foo.1.0.smithi186.mklmhy 2024-04-22T15:24:29.166 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi186.mklmhy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T15:24:29.166 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-22T15:24:29.166 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T15:24:29.166 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T15:24:29.167 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:29.167 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T15:24:29.167 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T15:24:29.167 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: Rados config object exists: conf-nfs.foo 2024-04-22T15:24:29.167 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: Creating key for client.nfs.foo.1.0.smithi186.mklmhy-rgw 2024-04-22T15:24:29.167 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi186.mklmhy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:24:29.167 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: Bind address in nfs.foo.1.0.smithi186.mklmhy's ganesha conf is defaulting to empty 2024-04-22T15:24:29.167 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:29.167 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: Deploying daemon nfs.foo.1.0.smithi186.mklmhy on smithi186 2024-04-22T15:24:29.167 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: from='client.34538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:24:29.167 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:28 smithi186 ceph-mon[49804]: pgmap v331: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 818 B/s wr, 3 op/s 2024-04-22T15:24:29.296 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:24:29.296 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (10m) 37s ago 19m 24.6M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:24:29.296 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (6m) 37s ago 19m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:24:29.296 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (6m) 13s ago 18m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (9m) 37s ago 18m 48.5M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (46s) 37s ago 13m 14.8M - 19.0.0-3223-g5883d494 3d983c26e278 5019782f86de 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (17s) 13s ago 13m 12.2M - 19.0.0-3223-g5883d494 3d983c26e278 6fe218feb080 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (9m) 37s ago 20m 513M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (8m) 13s ago 18m 453M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (8m) 37s ago 20m 61.5M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (7m) 13s ago 18m 49.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 starting - - - - 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (12m) 13s ago 12m 85.1M - 4.0 e1d6a67b021e 9a3e0bc009ed 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (9m) 37s ago 18m 11.2M - 1.3.1 1dbe0e931976 a7648f83607c 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (9m) 13s ago 18m 17.9M - 1.3.1 1dbe0e931976 fc4bb9d0c156 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (6m) 37s ago 17m 433M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (5m) 37s ago 17m 384M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (4m) 37s ago 16m 387M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:24:29.297 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (3m) 37s ago 16m 353M 1936M 19.0.0-3223-g5883d494 3d983c26e278 2378efa46eca 2024-04-22T15:24:29.298 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (3m) 13s ago 15m 378M 2704M 19.0.0-3223-g5883d494 3d983c26e278 9518e2eb7665 2024-04-22T15:24:29.298 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (2m) 13s ago 15m 347M 2704M 19.0.0-3223-g5883d494 3d983c26e278 e1c8f1367a89 2024-04-22T15:24:29.298 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (111s) 13s ago 15m 350M 2704M 19.0.0-3223-g5883d494 3d983c26e278 31e40cdb753c 2024-04-22T15:24:29.298 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (75s) 13s ago 14m 347M 2704M 19.0.0-3223-g5883d494 3d983c26e278 2f6a10f5d68a 2024-04-22T15:24:29.298 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (8m) 37s ago 17m 70.4M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:24:29.702 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:24:29.702 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:24:29.702 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:24:29.702 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:24:29.702 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:24:29.702 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:24:29.702 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:24:29.702 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:24:29.702 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:24:29.702 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:24:29.703 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:24:29.703 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:24:29.703 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:24:29.703 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:24:29.703 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 14 2024-04-22T15:24:29.703 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:24:29.703 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:24:29.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:29 smithi119 ceph-mon[63156]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:24:29.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:29 smithi119 ceph-mon[63156]: from='client.34546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:24:29.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:29 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/498563812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:30.020 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:29 smithi186 ceph-mon[49804]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:24:30.020 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:29 smithi186 ceph-mon[49804]: from='client.34546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:24:30.021 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:29 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/498563812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:30.053 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:24:30.053 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:24:30.053 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:24:30.053 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:24:30.053 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:24:30.053 INFO:teuthology.orchestra.run.smithi119.stdout: "osd", 2024-04-22T15:24:30.053 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:24:30.053 INFO:teuthology.orchestra.run.smithi119.stdout: "mds", 2024-04-22T15:24:30.053 INFO:teuthology.orchestra.run.smithi119.stdout: "crash", 2024-04-22T15:24:30.053 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr" 2024-04-22T15:24:30.054 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:24:30.054 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "16/23 daemons upgraded", 2024-04-22T15:24:30.054 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading nfs daemons", 2024-04-22T15:24:30.054 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:24:30.054 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:24:30.468 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:24:31.197 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:30 smithi186 ceph-mon[49804]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:24:31.197 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:30 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/587395165' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:24:31.197 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:30 smithi186 ceph-mon[49804]: pgmap v332: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 818 B/s wr, 3 op/s 2024-04-22T15:24:31.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:30 smithi119 ceph-mon[63156]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:24:31.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:30 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/587395165' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:24:31.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:30 smithi119 ceph-mon[63156]: pgmap v332: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 818 B/s wr, 3 op/s 2024-04-22T15:24:32.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:32.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:32.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:32 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:32.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:32.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:32.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:32 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:33.866 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:33 smithi119 ceph-mon[63156]: pgmap v333: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-04-22T15:24:33.931 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:33 smithi186 ceph-mon[49804]: pgmap v333: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-04-22T15:24:35.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:35.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:35.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:35 smithi119 ceph-mon[63156]: from='mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210]' entity='mds.foofs.smithi119.ncbqaf' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.186:0/2852469776"}]: dispatch 2024-04-22T15:24:35.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:35 smithi119 ceph-mon[63156]: pgmap v334: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-04-22T15:24:35.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:35.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:35.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:35 smithi186 ceph-mon[49804]: from='mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210]' entity='mds.foofs.smithi119.ncbqaf' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.186:0/2852469776"}]: dispatch 2024-04-22T15:24:35.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:35 smithi186 ceph-mon[49804]: pgmap v334: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-04-22T15:24:36.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:36 smithi119 ceph-mon[63156]: Evicting (and blocklisting) client session 24341 (172.21.15.186:0/2852469776) 2024-04-22T15:24:36.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:36 smithi119 ceph-mon[63156]: from='mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210]' entity='mds.foofs.smithi119.ncbqaf' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.186:0/2852469776"}]': finished 2024-04-22T15:24:36.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:36 smithi119 ceph-mon[63156]: osdmap e159: 8 total, 8 up, 8 in 2024-04-22T15:24:36.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:36.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:36 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:36 smithi186 ceph-mon[49804]: Evicting (and blocklisting) client session 24341 (172.21.15.186:0/2852469776) 2024-04-22T15:24:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:36 smithi186 ceph-mon[49804]: from='mds.? [v2:172.21.15.119:6834/2530880210,v1:172.21.15.119:6835/2530880210]' entity='mds.foofs.smithi119.ncbqaf' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.186:0/2852469776"}]': finished 2024-04-22T15:24:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:36 smithi186 ceph-mon[49804]: osdmap e159: 8 total, 8 up, 8 in 2024-04-22T15:24:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:36.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:36 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:37.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:37 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:37.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:37 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:37.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:37 smithi119 ceph-mon[63156]: pgmap v336: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.4 KiB/s wr, 7 op/s 2024-04-22T15:24:38.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:37 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:38.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:37 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:38.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:37 smithi186 ceph-mon[49804]: pgmap v336: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.4 KiB/s wr, 7 op/s 2024-04-22T15:24:38.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:38.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:38.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:38.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:38 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:39.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:38 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:39.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:38 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:39.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:38 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:39.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:38 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:39.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:39 smithi119 ceph-mon[63156]: pgmap v337: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.6 KiB/s wr, 4 op/s 2024-04-22T15:24:39.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:40.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:39 smithi186 ceph-mon[49804]: pgmap v337: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.6 KiB/s wr, 4 op/s 2024-04-22T15:24:40.068 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:39 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:40.939 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:40 smithi119 ceph-mon[63156]: pgmap v338: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.6 KiB/s wr, 4 op/s 2024-04-22T15:24:41.202 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:40 smithi186 ceph-mon[49804]: pgmap v338: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.6 KiB/s wr, 4 op/s 2024-04-22T15:24:42.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:42.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:42.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:42.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:24:42.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:42.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:42.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:42.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:42.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:42.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:42.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:42.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:42.994 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:42.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:42.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:42.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:42.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:42.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:42.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:42.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi119.zcjqol"}]: dispatch 2024-04-22T15:24:42.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi119.zcjqol"}]': finished 2024-04-22T15:24:42.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi186.mklmhy"}]: dispatch 2024-04-22T15:24:42.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi186.mklmhy"}]': finished 2024-04-22T15:24:42.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:42.995 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:42 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:43.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:43.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:43.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:43.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:24:43.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:43.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:43.026 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:43.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi119.zcjqol"}]: dispatch 2024-04-22T15:24:43.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi119.zcjqol"}]': finished 2024-04-22T15:24:43.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi186.mklmhy"}]: dispatch 2024-04-22T15:24:43.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi186.mklmhy"}]': finished 2024-04-22T15:24:43.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:43.027 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:42 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:43.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:43 smithi119 ceph-mon[63156]: Upgrade: Setting container_image for all nfs 2024-04-22T15:24:43.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:43 smithi119 ceph-mon[63156]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:24:43.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:43 smithi119 ceph-mon[63156]: pgmap v339: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-04-22T15:24:44.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:43 smithi186 ceph-mon[49804]: Upgrade: Setting container_image for all nfs 2024-04-22T15:24:44.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:43 smithi186 ceph-mon[49804]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:24:44.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:43 smithi186 ceph-mon[49804]: pgmap v339: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-04-22T15:24:44.939 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:44 smithi119 ceph-mon[63156]: Upgrade: Updating node-exporter.smithi119 (1/2) 2024-04-22T15:24:44.939 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:44 smithi119 ceph-mon[63156]: Deploying daemon node-exporter.smithi119 on smithi119 2024-04-22T15:24:45.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:44 smithi186 ceph-mon[49804]: Upgrade: Updating node-exporter.smithi119 (1/2) 2024-04-22T15:24:45.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:44 smithi186 ceph-mon[49804]: Deploying daemon node-exporter.smithi119 on smithi119 2024-04-22T15:24:45.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:45 smithi119 ceph-mon[63156]: pgmap v340: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-04-22T15:24:46.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:45 smithi186 ceph-mon[49804]: pgmap v340: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-04-22T15:24:47.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:46 smithi119 ceph-mon[63156]: pgmap v341: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 89 B/s rd, 358 B/s wr, 0 op/s 2024-04-22T15:24:47.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:46 smithi186 ceph-mon[49804]: pgmap v341: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 89 B/s rd, 358 B/s wr, 0 op/s 2024-04-22T15:24:48.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:48 smithi119 ceph-mon[63156]: pgmap v342: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-04-22T15:24:48.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:48.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:48 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:49.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:48 smithi186 ceph-mon[49804]: pgmap v342: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-04-22T15:24:49.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:49.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:48 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:50 smithi186 ceph-mon[49804]: Upgrade: Updating node-exporter.smithi186 (2/2) 2024-04-22T15:24:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:50 smithi186 ceph-mon[49804]: Deploying daemon node-exporter.smithi186 on smithi186 2024-04-22T15:24:51.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:50 smithi186 ceph-mon[49804]: pgmap v343: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:24:51.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:50 smithi119 ceph-mon[63156]: Upgrade: Updating node-exporter.smithi186 (2/2) 2024-04-22T15:24:51.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:50 smithi119 ceph-mon[63156]: Deploying daemon node-exporter.smithi186 on smithi186 2024-04-22T15:24:51.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:50 smithi119 ceph-mon[63156]: pgmap v343: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:24:53.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:52 smithi119 ceph-mon[63156]: pgmap v344: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:24:53.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:52 smithi186 ceph-mon[49804]: pgmap v344: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:24:55.002 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:54 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:55.153 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:54 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:56.150 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:55 smithi186 ceph-mon[49804]: pgmap v345: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:24:56.150 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:56.151 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:56.151 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:55 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:56.166 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:55 smithi119 ceph-mon[63156]: pgmap v345: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:24:56.167 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:56.167 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:56.167 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:55 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:58.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:57 smithi186 ceph-mon[49804]: pgmap v346: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:24:58.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:57 smithi119 ceph-mon[63156]: pgmap v346: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:24:59.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:59.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:59 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:59.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:24:59 smithi119 ceph-mon[63156]: pgmap v347: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:24:59.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:59.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:59 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:24:59.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:24:59 smithi186 ceph-mon[49804]: pgmap v347: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:25:00.741 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:00 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:00.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:00 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:00 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:00.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:00 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:00.847 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:25:01.546 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (10m) 0s ago 19m 24.7M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (7m) 0s ago 19m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (7m) 3s ago 19m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (10m) 0s ago 19m 55.2M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (78s) 0s ago 13m 37.1M - 19.0.0-3223-g5883d494 3d983c26e278 5019782f86de 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (49s) 3s ago 13m 12.4M - 19.0.0-3223-g5883d494 3d983c26e278 6fe218feb080 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (10m) 0s ago 20m 515M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (9m) 3s ago 18m 453M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (8m) 0s ago 21m 67.1M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (8m) 3s ago 18m 49.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (35s) 0s ago 13m 69.3M - 5.7 3d983c26e278 b02e7f98d060 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (30s) 3s ago 13m 69.2M - 5.7 3d983c26e278 bc27db3df7f2 2024-04-22T15:25:01.547 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (13s) 0s ago 19m 3464k - 1.5.0 0da6a335fe13 21ac95b78a9f 2024-04-22T15:25:01.548 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (6s) 3s ago 18m 3581k - 1.5.0 0da6a335fe13 c61c038d40f7 2024-04-22T15:25:01.548 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (6m) 0s ago 18m 455M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:25:01.548 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (5m) 0s ago 17m 390M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:25:01.548 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (4m) 0s ago 17m 306M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:25:01.548 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (4m) 0s ago 16m 367M 1936M 19.0.0-3223-g5883d494 3d983c26e278 2378efa46eca 2024-04-22T15:25:01.548 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (3m) 3s ago 16m 380M 2704M 19.0.0-3223-g5883d494 3d983c26e278 9518e2eb7665 2024-04-22T15:25:01.548 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (3m) 3s ago 16m 348M 2704M 19.0.0-3223-g5883d494 3d983c26e278 e1c8f1367a89 2024-04-22T15:25:01.548 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (2m) 3s ago 15m 351M 2704M 19.0.0-3223-g5883d494 3d983c26e278 31e40cdb753c 2024-04-22T15:25:01.548 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (107s) 3s ago 15m 349M 2704M 19.0.0-3223-g5883d494 3d983c26e278 2f6a10f5d68a 2024-04-22T15:25:01.548 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (9m) 0s ago 18m 70.5M - 2.33.4 514e6a882f6e 5b893d37011c 2024-04-22T15:25:01.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:01 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:01.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:01 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:01.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:01 smithi119 ceph-mon[63156]: pgmap v348: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:01.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:01 smithi119 ceph-mon[63156]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:01.995 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:25:01.995 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:25:01.995 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:25:01.995 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:25:01.995 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:25:01.995 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:25:01.996 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:25:01.996 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:25:01.996 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:25:01.996 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:25:01.996 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:25:01.996 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:25:01.996 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:25:01.996 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:25:01.996 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 14 2024-04-22T15:25:01.996 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:25:01.996 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:25:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:01 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:01 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:01 smithi186 ceph-mon[49804]: pgmap v348: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:02.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:01 smithi186 ceph-mon[49804]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:02.370 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:25:02.370 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: "osd", 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: "mds", 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: "crash", 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: "nfs" 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "18/23 daemons upgraded", 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading node-exporter daemons", 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:25:02.371 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:25:02.792 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:25:02.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:02 smithi119 ceph-mon[63156]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:02.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:02 smithi119 ceph-mon[63156]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:02.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:02 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:02.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:02 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:02.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:02 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/2760658945' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:03.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:02 smithi186 ceph-mon[49804]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:03.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:02 smithi186 ceph-mon[49804]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:03.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:02 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:03.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:02 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:03.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:02 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/2760658945' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:03.959 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:03 smithi119 ceph-mon[63156]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:03.959 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:03 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:03.959 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:03 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:03.959 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:03 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/1662522896' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:25:03.959 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:03 smithi119 ceph-mon[63156]: pgmap v349: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:04.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:03 smithi186 ceph-mon[49804]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:04.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:03 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:04.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:03 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:04.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:03 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/1662522896' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:25:04.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:03 smithi186 ceph-mon[49804]: pgmap v349: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:05.155 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:04 smithi119 ceph-mon[63156]: pgmap v350: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:05.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:04 smithi186 ceph-mon[49804]: pgmap v350: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:06.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:06.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:06 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:06.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:06.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:06.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:25:06.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:06.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:06.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:06.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:06.993 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:06 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:07.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:07 smithi119 ceph-mon[63156]: pgmap v351: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:08.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:07 smithi186 ceph-mon[49804]: pgmap v351: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:08.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:08 smithi119 ceph-mon[63156]: Upgrade: Updating prometheus.smithi119 2024-04-22T15:25:08.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:08 smithi119 ceph-mon[63156]: Deploying daemon prometheus.smithi119 on smithi119 2024-04-22T15:25:09.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:08 smithi186 ceph-mon[49804]: Upgrade: Updating prometheus.smithi119 2024-04-22T15:25:09.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:08 smithi186 ceph-mon[49804]: Deploying daemon prometheus.smithi119 on smithi119 2024-04-22T15:25:09.926 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:09 smithi186 ceph-mon[49804]: pgmap v352: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:25:09.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:09 smithi119 ceph-mon[63156]: pgmap v352: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:25:10.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:10 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:11.024 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:10 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:11.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:11 smithi119 ceph-mon[63156]: pgmap v353: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:12.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:11 smithi186 ceph-mon[49804]: pgmap v353: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:13.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:12 smithi119 ceph-mon[63156]: pgmap v354: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:13.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:12 smithi186 ceph-mon[49804]: pgmap v354: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:15.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:14 smithi119 ceph-mon[63156]: pgmap v355: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:15.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:14 smithi186 ceph-mon[49804]: pgmap v355: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:17.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:16 smithi119 ceph-mon[63156]: pgmap v356: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:16 smithi186 ceph-mon[49804]: pgmap v356: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:19.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:18 smithi119 ceph-mon[63156]: pgmap v357: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:25:19.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:18 smithi186 ceph-mon[49804]: pgmap v357: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:25:21.162 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:20 smithi119 ceph-mon[63156]: pgmap v358: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:21.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:20 smithi186 ceph-mon[49804]: pgmap v358: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:23.193 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:22 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:23.193 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:22 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:23.193 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:22 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:23.193 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:22 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:23.193 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:22 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:22 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:22 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:22 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:22 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:22 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:24.119 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:23 smithi119 ceph-mon[63156]: pgmap v359: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:24.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:23 smithi186 ceph-mon[49804]: pgmap v359: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:24.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:24 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:26.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:25 smithi119 ceph-mon[63156]: pgmap v360: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:25 smithi186 ceph-mon[49804]: pgmap v360: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:28.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:27 smithi119 ceph-mon[63156]: pgmap v361: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:28.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:27 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:28.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:27 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:28.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:27 smithi186 ceph-mon[49804]: pgmap v361: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:28.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:27 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:28.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:27 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:30.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:29 smithi119 ceph-mon[63156]: pgmap v362: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:25:30.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:29 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:30.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:29 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:30.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:29 smithi186 ceph-mon[49804]: pgmap v362: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:25:30.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:29 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:30.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:29 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:31.073 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:30 smithi119 ceph-mon[63156]: pgmap v363: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:30 smithi186 ceph-mon[49804]: pgmap v363: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:33.154 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:25:33.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:32 smithi119 ceph-mon[63156]: pgmap v364: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:33.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:32 smithi186 ceph-mon[49804]: pgmap v364: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:33.817 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:25:33.817 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (11m) 6s ago 20m 24.7M - 0.23.0 ba2b418f427c e994d1c2fbeb 2024-04-22T15:25:33.817 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (7m) 6s ago 20m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:25:33.817 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (7m) 35s ago 19m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:25:33.817 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (11m) 6s ago 19m 55.2M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:25:33.817 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (111s) 6s ago 14m 37.3M - 19.0.0-3223-g5883d494 3d983c26e278 5019782f86de 2024-04-22T15:25:33.817 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (81s) 35s ago 14m 12.4M - 19.0.0-3223-g5883d494 3d983c26e278 6fe218feb080 2024-04-22T15:25:33.817 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (10m) 6s ago 21m 517M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (9m) 35s ago 19m 453M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (9m) 6s ago 21m 65.1M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (8m) 35s ago 19m 49.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (68s) 6s ago 13m 69.4M - 5.7 3d983c26e278 b02e7f98d060 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (62s) 35s ago 13m 69.2M - 5.7 3d983c26e278 bc27db3df7f2 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (45s) 6s ago 19m 7884k - 1.5.0 0da6a335fe13 21ac95b78a9f 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (39s) 35s ago 19m 3581k - 1.5.0 0da6a335fe13 c61c038d40f7 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (7m) 6s ago 18m 455M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (6m) 6s ago 18m 391M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (5m) 6s ago 17m 307M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (4m) 6s ago 17m 368M 1936M 19.0.0-3223-g5883d494 3d983c26e278 2378efa46eca 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (4m) 35s ago 16m 380M 2704M 19.0.0-3223-g5883d494 3d983c26e278 9518e2eb7665 2024-04-22T15:25:33.818 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (3m) 35s ago 16m 348M 2704M 19.0.0-3223-g5883d494 3d983c26e278 e1c8f1367a89 2024-04-22T15:25:33.819 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (2m) 35s ago 16m 351M 2704M 19.0.0-3223-g5883d494 3d983c26e278 31e40cdb753c 2024-04-22T15:25:33.819 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (2m) 35s ago 15m 349M 2704M 19.0.0-3223-g5883d494 3d983c26e278 2f6a10f5d68a 2024-04-22T15:25:33.819 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (12s) 6s ago 19m 44.3M - 2.43.0 a07b618ecd1d 1637bea7bf74 2024-04-22T15:25:34.229 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:25:34.229 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:25:34.229 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:25:34.229 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:25:34.229 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:25:34.229 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:25:34.229 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:25:34.229 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:25:34.229 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:25:34.230 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:25:34.230 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:25:34.230 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:25:34.230 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:25:34.230 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:25:34.230 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 14 2024-04-22T15:25:34.230 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:25:34.230 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:25:34.230 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:33 smithi119 ceph-mon[63156]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:34.230 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:33 smithi119 ceph-mon[63156]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:34.230 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:33 smithi119 ceph-mon[63156]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:33 smithi186 ceph-mon[49804]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:33 smithi186 ceph-mon[49804]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:34.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:33 smithi186 ceph-mon[49804]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:34.591 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "osd", 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "mds", 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "crash", 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "nfs" 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "18/23 daemons upgraded", 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading alertmanager daemons", 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:25:34.592 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:25:35.016 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.126 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/1079513097' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: pgmap v365: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:35.127 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:35 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/1283218840' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:25:35.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:35.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:35.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:35.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:25:35.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/1079513097' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.526 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:35.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:25:35.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: pgmap v365: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:35.527 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:35 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/1283218840' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:25:37.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:36 smithi119 ceph-mon[63156]: Upgrade: Updating alertmanager.smithi119 2024-04-22T15:25:37.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:36 smithi119 ceph-mon[63156]: Deploying daemon alertmanager.smithi119 on smithi119 2024-04-22T15:25:37.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:36 smithi119 ceph-mon[63156]: pgmap v366: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:36 smithi186 ceph-mon[49804]: Upgrade: Updating alertmanager.smithi119 2024-04-22T15:25:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:36 smithi186 ceph-mon[49804]: Deploying daemon alertmanager.smithi119 on smithi119 2024-04-22T15:25:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:36 smithi186 ceph-mon[49804]: pgmap v366: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:39.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:38 smithi119 ceph-mon[63156]: pgmap v367: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:25:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:38 smithi186 ceph-mon[49804]: pgmap v367: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:25:40.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:40.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:39 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:41.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:40 smithi119 ceph-mon[63156]: pgmap v368: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:41.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:40 smithi186 ceph-mon[49804]: pgmap v368: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:42.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:42 smithi119 ceph-mon[63156]: pgmap v369: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:43.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:42 smithi186 ceph-mon[49804]: pgmap v369: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:45.151 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:44 smithi119 ceph-mon[63156]: pgmap v370: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:45.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:44 smithi186 ceph-mon[49804]: pgmap v370: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:47.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:47 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:47.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:47 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:47.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:47 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:47.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:47 smithi119 ceph-mon[63156]: pgmap v371: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:47 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:47 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:47 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:48.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:47 smithi186 ceph-mon[49804]: pgmap v371: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:49.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:48 smithi119 ceph-mon[63156]: pgmap v372: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:25:49.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:48 smithi186 ceph-mon[49804]: pgmap v372: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:25:51.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:50 smithi119 ceph-mon[63156]: pgmap v373: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:51.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:50 smithi186 ceph-mon[49804]: pgmap v373: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:53.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:52 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:53.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:52 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:53.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:52 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:53.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:52 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:53.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:53 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:53.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:53 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:53.992 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:53 smithi119 ceph-mon[63156]: pgmap v374: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:54.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:53 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:54.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:53 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:54.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:53 smithi186 ceph-mon[49804]: pgmap v374: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:55.154 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:54 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:55.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:54 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:55.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:55 smithi119 ceph-mon[63156]: pgmap v375: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:55 smithi186 ceph-mon[49804]: pgmap v375: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:57.831 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:57.831 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:57.831 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: pgmap v376: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi119:9093"}]: dispatch 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.832 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:57.833 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:57 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: pgmap v376: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi119:9093"}]: dispatch 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:58.277 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:57 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:25:59.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:58 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:25:59.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:58 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi119:9093"}]: dispatch 2024-04-22T15:25:59.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:58 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:25:59.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:58 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi119:9093"}]: dispatch 2024-04-22T15:26:00.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:59 smithi119 ceph-mon[63156]: Upgrade: Updating grafana.smithi119 2024-04-22T15:26:00.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:59 smithi119 ceph-mon[63156]: Deploying daemon grafana.smithi119 on smithi119 2024-04-22T15:26:00.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:25:59 smithi119 ceph-mon[63156]: pgmap v377: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:59 smithi186 ceph-mon[49804]: Upgrade: Updating grafana.smithi119 2024-04-22T15:26:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:59 smithi186 ceph-mon[49804]: Deploying daemon grafana.smithi119 on smithi119 2024-04-22T15:26:00.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:25:59 smithi186 ceph-mon[49804]: pgmap v377: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:01.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:00 smithi119 ceph-mon[63156]: pgmap v378: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:01.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:00 smithi186 ceph-mon[49804]: pgmap v378: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:03.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:02 smithi119 ceph-mon[63156]: pgmap v379: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:03.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:02 smithi186 ceph-mon[49804]: pgmap v379: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:05.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:04 smithi119 ceph-mon[63156]: pgmap v380: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:05.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:04 smithi186 ceph-mon[49804]: pgmap v380: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:05.383 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:26:06.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:05 smithi119 ceph-mon[63156]: from='client.34616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:26:06.025 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:05 smithi119 ceph-mon[63156]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:26:06.052 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:26:06.052 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (20s) 14s ago 20m 15.9M - 0.25.0 c8568f914cd2 c5227af9a99e 2024-04-22T15:26:06.052 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (8m) 14s ago 20m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:26:06.052 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (8m) 67s ago 20m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:26:06.052 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (11m) 14s ago 20m 54.6M - 8.3.5 dad864ee21e9 ee27b7f8f87e 2024-04-22T15:26:06.052 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (2m) 14s ago 14m 37.5M - 19.0.0-3223-g5883d494 3d983c26e278 5019782f86de 2024-04-22T15:26:06.052 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (113s) 67s ago 14m 12.4M - 19.0.0-3223-g5883d494 3d983c26e278 6fe218feb080 2024-04-22T15:26:06.052 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (11m) 14s ago 21m 517M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:26:06.052 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (10m) 67s ago 20m 453M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (9m) 14s ago 22m 65.9M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (9m) 67s ago 19m 49.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (100s) 14s ago 14m 70.6M - 5.7 3d983c26e278 b02e7f98d060 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (94s) 67s ago 14m 69.2M - 5.7 3d983c26e278 bc27db3df7f2 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (77s) 14s ago 20m 8770k - 1.5.0 0da6a335fe13 21ac95b78a9f 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (71s) 67s ago 19m 3581k - 1.5.0 0da6a335fe13 c61c038d40f7 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (7m) 14s ago 19m 456M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (6m) 14s ago 18m 391M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (6m) 14s ago 18m 306M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (5m) 14s ago 17m 369M 1936M 19.0.0-3223-g5883d494 3d983c26e278 2378efa46eca 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (4m) 67s ago 17m 380M 2704M 19.0.0-3223-g5883d494 3d983c26e278 9518e2eb7665 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (4m) 67s ago 17m 348M 2704M 19.0.0-3223-g5883d494 3d983c26e278 e1c8f1367a89 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (3m) 67s ago 16m 351M 2704M 19.0.0-3223-g5883d494 3d983c26e278 31e40cdb753c 2024-04-22T15:26:06.053 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (2m) 67s ago 16m 349M 2704M 19.0.0-3223-g5883d494 3d983c26e278 2f6a10f5d68a 2024-04-22T15:26:06.054 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (44s) 14s ago 19m 53.4M - 2.43.0 a07b618ecd1d 1637bea7bf74 2024-04-22T15:26:06.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:05 smithi186 ceph-mon[49804]: from='client.34616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:26:06.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:05 smithi186 ceph-mon[49804]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:26:06.463 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:26:06.464 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:26:06.464 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:26:06.464 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:26:06.464 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:26:06.465 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:26:06.465 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:26:06.465 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:26:06.465 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:26:06.465 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:26:06.465 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:26:06.466 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:26:06.466 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:26:06.466 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:26:06.466 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 14 2024-04-22T15:26:06.466 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:26:06.467 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:26:06.807 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:26:06.807 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:26:06.807 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": true, 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [ 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout: "osd", 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout: "mon", 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout: "mds", 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout: "crash", 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr", 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout: "nfs" 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": "18/23 daemons upgraded", 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "Currently upgrading grafana daemons", 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:26:06.808 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:26:07.146 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:06 smithi119 ceph-mon[63156]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:26:07.147 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:06 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/3852080836' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:07.147 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:06 smithi119 ceph-mon[63156]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:26:07.147 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:06 smithi119 ceph-mon[63156]: pgmap v381: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:07.224 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:26:07.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:06 smithi186 ceph-mon[49804]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:26:07.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:06 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/3852080836' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:07.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:06 smithi186 ceph-mon[49804]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:26:07.276 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:06 smithi186 ceph-mon[49804]: pgmap v381: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:08.159 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:07 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/436004433' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:26:08.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:07 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/436004433' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:26:09.176 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:08 smithi186 ceph-mon[49804]: pgmap v382: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:09.233 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:08 smithi119 ceph-mon[63156]: pgmap v382: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:10.204 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:10.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:11.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:11 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:11.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:11 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:11.491 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:11 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:26:11.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:11 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:11.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:11 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:11.492 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:11 smithi119 ceph-mon[63156]: pgmap v383: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:11.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:11 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:11.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:11 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:11.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:11 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:26:11.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:11 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:11.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:11 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:11.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:11 smithi186 ceph-mon[49804]: pgmap v383: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:13.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:12 smithi119 ceph-mon[63156]: pgmap v384: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:13.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:12 smithi186 ceph-mon[49804]: pgmap v384: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:15.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:14 smithi119 ceph-mon[63156]: pgmap v385: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:15.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:14 smithi186 ceph-mon[49804]: pgmap v385: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:17.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:16 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:17.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:16 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:17.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:16 smithi119 ceph-mon[63156]: pgmap v386: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:16 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:16 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:16 smithi186 ceph-mon[49804]: pgmap v386: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:18.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:17 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:18.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:17 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:18.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:17 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:18.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:17 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:19.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:18 smithi119 ceph-mon[63156]: pgmap v387: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:19.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:18 smithi186 ceph-mon[49804]: pgmap v387: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:26:21.742 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi119:3000"}]: dispatch 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi119:3000"}]: dispatch 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.743 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: Upgrade: Finalizing container_image settings 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:26:21.744 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.745 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.746 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: Upgrade: Complete! 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.747 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.748 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:21 smithi119 ceph-mon[63156]: pgmap v388: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:21.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:26:21.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:26:21.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:26:21.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:26:21.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi119:3000"}]: dispatch 2024-04-22T15:26:21.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi119:3000"}]: dispatch 2024-04-22T15:26:21.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.776 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.777 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:21.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: Upgrade: Finalizing container_image settings 2024-04-22T15:26:21.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:26:21.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:26:21.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:26:21.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:26:21.778 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:26:21.779 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: Upgrade: Complete! 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:26:21.780 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:26:21.781 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.781 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.781 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:26:21.781 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:26:21.781 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:26:21.781 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.781 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:21.781 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:21 smithi186 ceph-mon[49804]: pgmap v388: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:23.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:22 smithi119 ceph-mon[63156]: pgmap v389: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:22 smithi186 ceph-mon[49804]: pgmap v389: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:25.155 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:24 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:26.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:25 smithi119 ceph-mon[63156]: pgmap v390: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:26.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:25 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:25 smithi186 ceph-mon[49804]: pgmap v390: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:25 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:26:27.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:26 smithi119 ceph-mon[63156]: pgmap v391: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:27.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:26 smithi186 ceph-mon[49804]: pgmap v391: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:29.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:28 smithi119 ceph-mon[63156]: pgmap v392: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:29.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:28 smithi186 ceph-mon[49804]: pgmap v392: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:31.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:30 smithi119 ceph-mon[63156]: pgmap v393: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:30 smithi186 ceph-mon[49804]: pgmap v393: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:33.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:32 smithi119 ceph-mon[63156]: pgmap v394: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:33.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:32 smithi186 ceph-mon[49804]: pgmap v394: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:35.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:34 smithi119 ceph-mon[63156]: pgmap v395: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:34 smithi186 ceph-mon[49804]: pgmap v395: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:37.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:36 smithi119 ceph-mon[63156]: pgmap v396: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:36 smithi186 ceph-mon[49804]: pgmap v396: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:38.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:37 smithi119 ceph-mon[63156]: from='client.34640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:26:38.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:37 smithi186 ceph-mon[49804]: from='client.34640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:26:39.176 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:38 smithi186 ceph-mon[49804]: pgmap v397: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:39.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:38 smithi119 ceph-mon[63156]: pgmap v397: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:40.238 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:40.267 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:26:40.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:39 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:41.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:40 smithi119 ceph-mon[63156]: pgmap v398: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:41.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:40 smithi186 ceph-mon[49804]: pgmap v398: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:42.328 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:26:42.328 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (56s) 26s ago 21m 15.9M - 0.25.0 c8568f914cd2 c5227af9a99e 2024-04-22T15:26:42.328 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (8m) 26s ago 21m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:26:42.328 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (8m) 103s ago 20m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:26:42.328 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (32s) 26s ago 21m 48.5M - 9.4.12 2bacad6d85d8 77bf655d5c44 2024-04-22T15:26:42.328 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (2m) 26s ago 15m 37.7M - 19.0.0-3223-g5883d494 3d983c26e278 5019782f86de 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (2m) 103s ago 15m 12.4M - 19.0.0-3223-g5883d494 3d983c26e278 6fe218feb080 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (11m) 26s ago 22m 524M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (10m) 103s ago 20m 453M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (10m) 26s ago 22m 69.6M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (9m) 103s ago 20m 49.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (2m) 26s ago 15m 70.6M - 5.7 3d983c26e278 b02e7f98d060 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (2m) 103s ago 14m 69.2M - 5.7 3d983c26e278 bc27db3df7f2 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (113s) 26s ago 20m 8196k - 1.5.0 0da6a335fe13 21ac95b78a9f 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (107s) 103s ago 20m 3581k - 1.5.0 0da6a335fe13 c61c038d40f7 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (8m) 26s ago 19m 456M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (7m) 26s ago 19m 391M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (6m) 26s ago 18m 307M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (6m) 26s ago 18m 369M 1936M 19.0.0-3223-g5883d494 3d983c26e278 2378efa46eca 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (5m) 103s ago 18m 380M 2704M 19.0.0-3223-g5883d494 3d983c26e278 9518e2eb7665 2024-04-22T15:26:42.329 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (4m) 103s ago 17m 348M 2704M 19.0.0-3223-g5883d494 3d983c26e278 e1c8f1367a89 2024-04-22T15:26:42.330 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (4m) 103s ago 17m 351M 2704M 19.0.0-3223-g5883d494 3d983c26e278 31e40cdb753c 2024-04-22T15:26:42.330 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (3m) 103s ago 16m 349M 2704M 19.0.0-3223-g5883d494 3d983c26e278 2f6a10f5d68a 2024-04-22T15:26:42.330 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (81s) 26s ago 20m 53.6M - 2.43.0 a07b618ecd1d 1637bea7bf74 2024-04-22T15:26:42.969 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions' 2024-04-22T15:26:43.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:42 smithi119 ceph-mon[63156]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:26:43.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:42 smithi119 ceph-mon[63156]: pgmap v399: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:43.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:42 smithi186 ceph-mon[49804]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:26:43.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:42 smithi186 ceph-mon[49804]: pgmap v399: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:45.121 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:26:45.121 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:26:45.121 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:26:45.121 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:26:45.121 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:26:45.121 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:26:45.121 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:26:45.121 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:26:45.121 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:26:45.122 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:26:45.122 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:26:45.122 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:26:45.122 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:26:45.122 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:26:45.122 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 14 2024-04-22T15:26:45.122 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:26:45.122 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:26:45.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:44 smithi119 ceph-mon[63156]: pgmap v400: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:45.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:44 smithi186 ceph-mon[49804]: pgmap v400: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:45.803 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-22T15:26:46.108 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:45 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/3686208908' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:46.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:45 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/3686208908' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:26:47.236 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:46 smithi119 ceph-mon[63156]: pgmap v401: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:47.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:46 smithi186 ceph-mon[49804]: pgmap v401: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:47.619 INFO:teuthology.orchestra.run.smithi119.stdout:wait for servicemap items w/ changing names to refresh 2024-04-22T15:26:48.264 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'sleep 60' 2024-04-22T15:26:49.115 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:48 smithi119 ceph-mon[63156]: pgmap v402: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:49.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:48 smithi186 ceph-mon[49804]: pgmap v402: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:51.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:50 smithi119 ceph-mon[63156]: pgmap v403: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:51.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:50 smithi186 ceph-mon[49804]: pgmap v403: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:53.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:52 smithi119 ceph-mon[63156]: pgmap v404: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:53.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:52 smithi186 ceph-mon[49804]: pgmap v404: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:55.155 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:54 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:55.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:54 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:56.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:55 smithi119 ceph-mon[63156]: pgmap v405: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:55 smithi186 ceph-mon[49804]: pgmap v405: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:57.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:56 smithi119 ceph-mon[63156]: pgmap v406: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:57.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:56 smithi186 ceph-mon[49804]: pgmap v406: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:26:59.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:26:58 smithi119 ceph-mon[63156]: pgmap v407: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:26:59.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:26:58 smithi186 ceph-mon[49804]: pgmap v407: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:27:01.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:00 smithi119 ceph-mon[63156]: pgmap v408: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:01.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:00 smithi186 ceph-mon[49804]: pgmap v408: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:03.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:02 smithi119 ceph-mon[63156]: pgmap v409: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:03.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:02 smithi186 ceph-mon[49804]: pgmap v409: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:05.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:04 smithi119 ceph-mon[63156]: pgmap v410: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:05.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:04 smithi186 ceph-mon[49804]: pgmap v410: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:07.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:06 smithi119 ceph-mon[63156]: pgmap v411: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:07.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:06 smithi186 ceph-mon[49804]: pgmap v411: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:09.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:08 smithi119 ceph-mon[63156]: pgmap v412: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:27:09.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:08 smithi186 ceph-mon[49804]: pgmap v412: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:27:10.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:10.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:11.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:10 smithi119 ceph-mon[63156]: pgmap v413: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:11.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:10 smithi186 ceph-mon[49804]: pgmap v413: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:13.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:12 smithi119 ceph-mon[63156]: pgmap v414: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:13.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:12 smithi186 ceph-mon[49804]: pgmap v414: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:15.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:14 smithi119 ceph-mon[63156]: pgmap v415: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:15.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:14 smithi186 ceph-mon[49804]: pgmap v415: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:17.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:16 smithi119 ceph-mon[63156]: pgmap v416: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:16 smithi186 ceph-mon[49804]: pgmap v416: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:19.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:18 smithi119 ceph-mon[63156]: pgmap v417: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:27:19.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:18 smithi186 ceph-mon[49804]: pgmap v417: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:27:21.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:20 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:27:21.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:20 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:27:22.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:21 smithi119 ceph-mon[63156]: pgmap v418: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:22.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:27:22.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:27:22.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:27:22.242 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:21 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:27:22.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:21 smithi186 ceph-mon[49804]: pgmap v418: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:22.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:27:22.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:27:22.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:27:22.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:21 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' 2024-04-22T15:27:23.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:22 smithi119 ceph-mon[63156]: pgmap v419: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:23.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:22 smithi186 ceph-mon[49804]: pgmap v419: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:25.154 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:24 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:25.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:24 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:26.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:25 smithi119 ceph-mon[63156]: pgmap v420: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:26.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:25 smithi186 ceph-mon[49804]: pgmap v420: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:27.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:26 smithi119 ceph-mon[63156]: pgmap v421: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:27.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:26 smithi186 ceph-mon[49804]: pgmap v421: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:29.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:28 smithi119 ceph-mon[63156]: pgmap v422: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:27:29.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:28 smithi186 ceph-mon[49804]: pgmap v422: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:27:31.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:30 smithi119 ceph-mon[63156]: pgmap v423: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:31.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:30 smithi186 ceph-mon[49804]: pgmap v423: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:33.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:32 smithi119 ceph-mon[63156]: pgmap v424: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:33.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:32 smithi186 ceph-mon[49804]: pgmap v424: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:35.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:34 smithi119 ceph-mon[63156]: pgmap v425: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:35.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:34 smithi186 ceph-mon[49804]: pgmap v425: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:37.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:36 smithi119 ceph-mon[63156]: pgmap v426: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:37.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:36 smithi186 ceph-mon[49804]: pgmap v426: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:39.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:38 smithi119 ceph-mon[63156]: pgmap v427: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:27:39.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:38 smithi186 ceph-mon[49804]: pgmap v427: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:27:40.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:39 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:40.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:39 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:41.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:40 smithi119 ceph-mon[63156]: pgmap v428: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:41.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:40 smithi186 ceph-mon[49804]: pgmap v428: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:43.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:42 smithi119 ceph-mon[63156]: pgmap v429: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:43.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:42 smithi186 ceph-mon[49804]: pgmap v429: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:45.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:44 smithi119 ceph-mon[63156]: pgmap v430: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:45.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:44 smithi186 ceph-mon[49804]: pgmap v430: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:47.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:46 smithi119 ceph-mon[63156]: pgmap v431: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:47.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:46 smithi186 ceph-mon[49804]: pgmap v431: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:49.852 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:49 smithi186 ceph-mon[49804]: pgmap v432: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:27:49.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:49 smithi119 ceph-mon[63156]: pgmap v432: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:27:50.293 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:27:51.163 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:50 smithi119 ceph-mon[63156]: pgmap v433: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:51.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:50 smithi186 ceph-mon[49804]: pgmap v433: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:52.104 INFO:teuthology.orchestra.run.smithi119.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:27:52.105 INFO:teuthology.orchestra.run.smithi119.stdout:alertmanager.smithi119 smithi119 *:9093,9094 running (2m) 96s ago 22m 15.9M - 0.25.0 c8568f914cd2 c5227af9a99e 2024-04-22T15:27:52.105 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi119 smithi119 running (10m) 96s ago 22m 6660k - 19.0.0-3223-g5883d494 3d983c26e278 5bfb22414ff3 2024-04-22T15:27:52.105 INFO:teuthology.orchestra.run.smithi119.stdout:crash.smithi186 smithi186 running (10m) 2m ago 21m 6665k - 19.0.0-3223-g5883d494 3d983c26e278 d7c35b0c64de 2024-04-22T15:27:52.105 INFO:teuthology.orchestra.run.smithi119.stdout:grafana.smithi119 smithi119 *:3000 running (102s) 96s ago 22m 48.5M - 9.4.12 2bacad6d85d8 77bf655d5c44 2024-04-22T15:27:52.105 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi119.ncbqaf smithi119 running (4m) 96s ago 16m 37.7M - 19.0.0-3223-g5883d494 3d983c26e278 5019782f86de 2024-04-22T15:27:52.105 INFO:teuthology.orchestra.run.smithi119.stdout:mds.foofs.smithi186.dglszf smithi186 running (3m) 2m ago 16m 12.4M - 19.0.0-3223-g5883d494 3d983c26e278 6fe218feb080 2024-04-22T15:27:52.105 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi119.jjbxxd smithi119 *:8443,9283,8765 running (12m) 96s ago 23m 524M - 19.0.0-3223-g5883d494 3d983c26e278 16467215bb71 2024-04-22T15:27:52.105 INFO:teuthology.orchestra.run.smithi119.stdout:mgr.smithi186.vtjgwg smithi186 *:8443,9283,8765 running (11m) 2m ago 21m 453M - 19.0.0-3223-g5883d494 3d983c26e278 6ae3abd5a2e9 2024-04-22T15:27:52.105 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi119 smithi119 running (11m) 96s ago 23m 69.6M 2048M 19.0.0-3223-g5883d494 3d983c26e278 6f921c4637d2 2024-04-22T15:27:52.105 INFO:teuthology.orchestra.run.smithi119.stdout:mon.smithi186 smithi186 running (11m) 2m ago 21m 49.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38867a5eb7be 2024-04-22T15:27:52.105 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.0.0.smithi119.zcjqol smithi119 *:2049 running (3m) 96s ago 16m 70.6M - 5.7 3d983c26e278 b02e7f98d060 2024-04-22T15:27:52.106 INFO:teuthology.orchestra.run.smithi119.stdout:nfs.foo.1.0.smithi186.mklmhy smithi186 *:2049 running (3m) 2m ago 16m 69.2M - 5.7 3d983c26e278 bc27db3df7f2 2024-04-22T15:27:52.106 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi119 smithi119 *:9100 running (3m) 96s ago 22m 8196k - 1.5.0 0da6a335fe13 21ac95b78a9f 2024-04-22T15:27:52.106 INFO:teuthology.orchestra.run.smithi119.stdout:node-exporter.smithi186 smithi186 *:9100 running (2m) 2m ago 21m 3581k - 1.5.0 0da6a335fe13 c61c038d40f7 2024-04-22T15:27:52.106 INFO:teuthology.orchestra.run.smithi119.stdout:osd.0 smithi119 running (9m) 96s ago 20m 456M 1936M 19.0.0-3223-g5883d494 3d983c26e278 b35da6e9f2fa 2024-04-22T15:27:52.106 INFO:teuthology.orchestra.run.smithi119.stdout:osd.1 smithi119 running (8m) 96s ago 20m 391M 1936M 19.0.0-3223-g5883d494 3d983c26e278 8e40a47e7598 2024-04-22T15:27:52.106 INFO:teuthology.orchestra.run.smithi119.stdout:osd.2 smithi119 running (7m) 96s ago 20m 307M 1936M 19.0.0-3223-g5883d494 3d983c26e278 14287157a66e 2024-04-22T15:27:52.106 INFO:teuthology.orchestra.run.smithi119.stdout:osd.3 smithi119 running (7m) 96s ago 19m 369M 1936M 19.0.0-3223-g5883d494 3d983c26e278 2378efa46eca 2024-04-22T15:27:52.106 INFO:teuthology.orchestra.run.smithi119.stdout:osd.4 smithi186 running (6m) 2m ago 19m 380M 2704M 19.0.0-3223-g5883d494 3d983c26e278 9518e2eb7665 2024-04-22T15:27:52.106 INFO:teuthology.orchestra.run.smithi119.stdout:osd.5 smithi186 running (5m) 2m ago 18m 348M 2704M 19.0.0-3223-g5883d494 3d983c26e278 e1c8f1367a89 2024-04-22T15:27:52.106 INFO:teuthology.orchestra.run.smithi119.stdout:osd.6 smithi186 running (5m) 2m ago 18m 351M 2704M 19.0.0-3223-g5883d494 3d983c26e278 31e40cdb753c 2024-04-22T15:27:52.106 INFO:teuthology.orchestra.run.smithi119.stdout:osd.7 smithi186 running (4m) 2m ago 18m 349M 2704M 19.0.0-3223-g5883d494 3d983c26e278 2f6a10f5d68a 2024-04-22T15:27:52.106 INFO:teuthology.orchestra.run.smithi119.stdout:prometheus.smithi119 smithi119 *:9095 running (2m) 96s ago 21m 53.6M - 2.43.0 a07b618ecd1d 1637bea7bf74 2024-04-22T15:27:52.753 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade status' 2024-04-22T15:27:52.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:52 smithi119 ceph-mon[63156]: from='client.34652 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:27:52.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:52 smithi119 ceph-mon[63156]: pgmap v434: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:53.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:52 smithi186 ceph-mon[49804]: from='client.34652 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:27:53.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:52 smithi186 ceph-mon[49804]: pgmap v434: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:54.745 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:27:54.745 INFO:teuthology.orchestra.run.smithi119.stdout: "target_image": null, 2024-04-22T15:27:54.745 INFO:teuthology.orchestra.run.smithi119.stdout: "in_progress": false, 2024-04-22T15:27:54.745 INFO:teuthology.orchestra.run.smithi119.stdout: "which": "", 2024-04-22T15:27:54.745 INFO:teuthology.orchestra.run.smithi119.stdout: "services_complete": [], 2024-04-22T15:27:54.745 INFO:teuthology.orchestra.run.smithi119.stdout: "progress": null, 2024-04-22T15:27:54.745 INFO:teuthology.orchestra.run.smithi119.stdout: "message": "", 2024-04-22T15:27:54.745 INFO:teuthology.orchestra.run.smithi119.stdout: "is_paused": false 2024-04-22T15:27:54.745 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:27:55.058 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:54 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:55.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:54 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:55.385 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph health detail' 2024-04-22T15:27:55.942 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:55 smithi119 ceph-mon[63156]: from='client.34656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:27:55.942 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:55 smithi119 ceph-mon[63156]: pgmap v435: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:55 smithi186 ceph-mon[49804]: from='client.34656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:27:56.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:55 smithi186 ceph-mon[49804]: pgmap v435: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:57.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:56 smithi119 ceph-mon[63156]: pgmap v436: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:57.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:56 smithi186 ceph-mon[49804]: pgmap v436: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:27:57.410 INFO:teuthology.orchestra.run.smithi119.stdout:HEALTH_OK 2024-04-22T15:27:57.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:57 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/503123035' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:27:58.019 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions' 2024-04-22T15:27:58.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:57 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/503123035' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:27:59.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:27:59 smithi119 ceph-mon[63156]: pgmap v437: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:28:00.003 INFO:teuthology.orchestra.run.smithi119.stdout:{ 2024-04-22T15:28:00.004 INFO:teuthology.orchestra.run.smithi119.stdout: "mon": { 2024-04-22T15:28:00.004 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:28:00.004 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:28:00.004 INFO:teuthology.orchestra.run.smithi119.stdout: "mgr": { 2024-04-22T15:28:00.004 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:28:00.004 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:28:00.004 INFO:teuthology.orchestra.run.smithi119.stdout: "osd": { 2024-04-22T15:28:00.004 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:28:00.004 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:28:00.004 INFO:teuthology.orchestra.run.smithi119.stdout: "mds": { 2024-04-22T15:28:00.004 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:28:00.004 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:28:00.005 INFO:teuthology.orchestra.run.smithi119.stdout: "overall": { 2024-04-22T15:28:00.005 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 14 2024-04-22T15:28:00.005 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:28:00.005 INFO:teuthology.orchestra.run.smithi119.stdout:} 2024-04-22T15:28:00.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:27:59 smithi186 ceph-mon[49804]: pgmap v437: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:28:00.607 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:00 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/1316291505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:28:00.608 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-22T15:28:01.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:00 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/1316291505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:28:01.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:01 smithi186 ceph-mon[49804]: pgmap v438: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:01.896 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:01 smithi119 ceph-mon[63156]: pgmap v438: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:02.604 INFO:teuthology.orchestra.run.smithi119.stdout:true 2024-04-22T15:28:02.916 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:02 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/2508363593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:28:03.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:02 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/2508363593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:28:03.256 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-22T15:28:03.894 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:03 smithi119 ceph-mon[63156]: pgmap v439: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:04.025 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:03 smithi186 ceph-mon[49804]: pgmap v439: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:05.154 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:04 smithi119 ceph-mon[63156]: pgmap v440: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:05.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:04 smithi186 ceph-mon[49804]: pgmap v440: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:05.294 INFO:teuthology.orchestra.run.smithi119.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)" 2024-04-22T15:28:05.965 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-04-22T15:28:05.966 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-04-22T15:28:05.966 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-22T15:28:06.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:05 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/261088421' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:28:06.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:05 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/261088421' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:28:07.205 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:06 smithi119 ceph-mon[63156]: pgmap v441: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:07.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:06 smithi186 ceph-mon[49804]: pgmap v441: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:07.963 INFO:teuthology.orchestra.run.smithi119.stdout: 2024-04-22T15:28:07.963 INFO:teuthology.orchestra.run.smithi119.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-22T15:04:44.298348Z", "last_refresh": "2024-04-22T15:26:15.914161Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-22T15:04:37.826787Z", "last_refresh": "2024-04-22T15:24:58.343100Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-04-22T15:04:41.840104Z", "last_refresh": "2024-04-22T15:26:15.914344Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-22T15:11:14.762937Z", "last_refresh": "2024-04-22T15:24:58.343752Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-22T15:04:35.847067Z", "last_refresh": "2024-04-22T15:24:58.343208Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi119:172.21.15.119=smithi119", "smithi186:172.21.15.186=smithi186"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-22T15:05:26.289192Z", "last_refresh": "2024-04-22T15:24:58.343293Z", "running": 2, "size": 2}}, {"events": ["2024-04-22T15:27:21.385213Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-04-22T15:11:34.733360Z", "last_refresh": "2024-04-22T15:24:58.343827Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-22T15:04:43.132534Z", "last_refresh": "2024-04-22T15:24:58.343377Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", "last_refresh": "2024-04-22T15:24:58.343455Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-22T15:04:39.724425Z", "last_refresh": "2024-04-22T15:26:15.914507Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-22T15:28:08.595 INFO:tasks.cephadm:nfs.foo has 2/2 2024-04-22T15:28:08.595 DEBUG:teuthology.parallel:result is None 2024-04-22T15:28:08.595 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-22T15:28:08.605 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T15:28:08.605 DEBUG:teuthology.orchestra.run.smithi119:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-04-22T15:28:08.629 INFO:teuthology.orchestra.run.smithi119.stderr:+ umount /mnt/foo 2024-04-22T15:28:08.641 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T15:28:08.649 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T15:28:08.649 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-04-22T15:28:09.233 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:08 smithi119 ceph-mon[63156]: from='client.34676 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:28:09.233 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:08 smithi119 ceph-mon[63156]: pgmap v442: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:28:09.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:08 smithi186 ceph-mon[49804]: from='client.34676 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:28:09.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:08 smithi186 ceph-mon[49804]: pgmap v442: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:28:09.970 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:09 smithi119 ceph-mon[63156]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:10.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:09 smithi186 ceph-mon[49804]: from='mgr.24595 172.21.15.119:0/4186441529' entity='mgr.smithi119.jjbxxd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:10.698 INFO:teuthology.orchestra.run.smithi119.stdout: "foo" 2024-04-22T15:28:10.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:10 smithi119 ceph-mon[63156]: from='client.34680 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:28:10.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:10 smithi119 ceph-mon[63156]: pgmap v443: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:11.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:10 smithi186 ceph-mon[49804]: from='client.34680 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:28:11.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:10 smithi186 ceph-mon[49804]: pgmap v443: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:11.344 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-04-22T15:28:13.122 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:12 smithi119 ceph-mon[63156]: pgmap v444: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:13.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:12 smithi186 ceph-mon[49804]: pgmap v444: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:13.365 INFO:teuthology.orchestra.run.smithi119.stdout:[ 2024-04-22T15:28:13.365 INFO:teuthology.orchestra.run.smithi119.stdout: { 2024-04-22T15:28:13.365 INFO:teuthology.orchestra.run.smithi119.stdout: "access_type": "RW", 2024-04-22T15:28:13.365 INFO:teuthology.orchestra.run.smithi119.stdout: "clients": [], 2024-04-22T15:28:13.365 INFO:teuthology.orchestra.run.smithi119.stdout: "cluster_id": "foo", 2024-04-22T15:28:13.365 INFO:teuthology.orchestra.run.smithi119.stdout: "export_id": 1, 2024-04-22T15:28:13.365 INFO:teuthology.orchestra.run.smithi119.stdout: "fsal": { 2024-04-22T15:28:13.365 INFO:teuthology.orchestra.run.smithi119.stdout: "fs_name": "foofs", 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: "name": "CEPH", 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: "user_id": "nfs.foo.1" 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: }, 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: "path": "/", 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: "protocols": [ 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: 4 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: ], 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: "pseudo": "/fake", 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: "security_label": true, 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: "squash": "none", 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: "transports": [ 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: "TCP" 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: ] 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout: } 2024-04-22T15:28:13.366 INFO:teuthology.orchestra.run.smithi119.stdout:] 2024-04-22T15:28:13.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:13 smithi119 ceph-mon[63156]: from='client.34684 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:28:14.026 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-04-22T15:28:14.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:13 smithi186 ceph-mon[49804]: from='client.34684 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:28:15.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:14 smithi119 ceph-mon[63156]: pgmap v445: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:15.274 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:14 smithi186 ceph-mon[49804]: pgmap v445: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:15.788 INFO:teuthology.orchestra.run.smithi119.stdout:foo rec-0000000000000004:nfs.foo.1 2024-04-22T15:28:15.788 INFO:teuthology.orchestra.run.smithi119.stdout:foo grace 2024-04-22T15:28:15.788 INFO:teuthology.orchestra.run.smithi119.stdout:foo rec-0000000000000004:nfs.foo.0 2024-04-22T15:28:15.788 INFO:teuthology.orchestra.run.smithi119.stdout:foo conf-nfs.foo 2024-04-22T15:28:15.788 INFO:teuthology.orchestra.run.smithi119.stdout:foo export-1 2024-04-22T15:28:16.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:15 smithi119 ceph-mon[63156]: mgrmap e43: smithi119.jjbxxd(active, since 11m), standbys: smithi186.vtjgwg 2024-04-22T15:28:16.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:15 smithi186 ceph-mon[49804]: mgrmap e43: smithi119.jjbxxd(active, since 11m), standbys: smithi186.vtjgwg 2024-04-22T15:28:16.377 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T15:28:16.386 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T15:28:16.387 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 74739db6-00b9-11ef-bc93-c7b262605968 -- bash -c 'set -ex 2024-04-22T15:28:16.387 DEBUG:teuthology.orchestra.run.smithi119:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-04-22T15:28:16.387 DEBUG:teuthology.orchestra.run.smithi119:> ' 2024-04-22T15:28:17.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:16 smithi119 ceph-mon[63156]: pgmap v446: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:17.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:16 smithi186 ceph-mon[49804]: pgmap v446: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:28:18.064 INFO:teuthology.orchestra.run.smithi119.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-04-22T15:28:18.287 INFO:teuthology.orchestra.run.smithi119.stderr:+ [[ 6 -gt 2 ]] 2024-04-22T15:28:18.576 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:18 smithi119 ceph-mon[63156]: from='client.? 172.21.15.119:0/1654391047' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-22T15:28:18.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:18 smithi186 ceph-mon[49804]: from='client.? 172.21.15.119:0/1654391047' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-22T15:28:18.902 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-22T15:28:18.912 INFO:tasks.cephadm:Teardown begin 2024-04-22T15:28:18.913 DEBUG:teuthology.orchestra.run.smithi119:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:28:18.939 DEBUG:teuthology.orchestra.run.smithi186:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:28:18.971 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-22T15:28:18.971 DEBUG:teuthology.orchestra.run.smithi119:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-22T15:28:18.994 DEBUG:teuthology.orchestra.run.smithi186:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-22T15:28:19.029 INFO:tasks.cephadm:Stopping all daemons... 2024-04-22T15:28:19.029 INFO:tasks.cephadm.mon.smithi119:Stopping mon.smithi119... 2024-04-22T15:28:19.029 DEBUG:teuthology.orchestra.run.smithi119:> sudo systemctl stop ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi119 2024-04-22T15:28:19.241 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:19 smithi119 systemd[1]: Stopping Ceph mon.smithi119 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:28:19.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:19 smithi119 ceph-mon[63156]: pgmap v447: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T15:28:19.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:19 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119[63152]: 2024-04-22T15:28:19.352+0000 7f6b1eeb0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi119 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:28:19.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:19 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119[63152]: 2024-04-22T15:28:19.352+0000 7f6b1eeb0700 -1 mon.smithi119@0(leader) e3 *** Got Signal Terminated *** 2024-04-22T15:28:19.674 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:19 smithi119 podman[109155]: 2024-04-22 15:28:19.394381421 +0000 UTC m=+0.176727144 container died 6f921c4637d27c079d3cc73d868bbb8a405800f988e04b8b4c34785f12447ffe (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-22T15:28:19.775 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:19 smithi186 ceph-mon[49804]: pgmap v447: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T15:28:19.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:19 smithi119 podman[109155]: 2024-04-22 15:28:19.779087501 +0000 UTC m=+0.561433223 container cleanup 6f921c4637d27c079d3cc73d868bbb8a405800f988e04b8b4c34785f12447ffe (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:28:19.991 INFO:journalctl@ceph.mon.smithi119.smithi119.stdout:Apr 22 15:28:19 smithi119 bash[109155]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi119 2024-04-22T15:28:20.200 DEBUG:teuthology.orchestra.run.smithi119:> sudo pkill -f 'journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi119.service' 2024-04-22T15:28:20.247 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:28:20.247 INFO:tasks.cephadm.mon.smithi119:Stopped mon.smithi119 2024-04-22T15:28:20.247 INFO:tasks.cephadm.mon.smithi186:Stopping mon.smithi186... 2024-04-22T15:28:20.247 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi186 2024-04-22T15:28:20.525 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:20 smithi186 systemd[1]: Stopping Ceph mon.smithi186 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:28:20.814 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:20 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186[49800]: 2024-04-22T15:28:20.553+0000 7fc7e7494700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi186 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:28:20.815 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:20 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186[49800]: 2024-04-22T15:28:20.553+0000 7fc7e7494700 -1 mon.smithi186@1(peon) e3 *** Got Signal Terminated *** 2024-04-22T15:28:20.815 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:20 smithi186 podman[74938]: 2024-04-22 15:28:20.574121293 +0000 UTC m=+0.141245576 container died 38867a5eb7be29953953fe64f1bac4c443ee60df22483bc840632ce6c0d552f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:28:21.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:21 smithi186 podman[74938]: 2024-04-22 15:28:21.020210177 +0000 UTC m=+0.587334452 container cleanup 38867a5eb7be29953953fe64f1bac4c443ee60df22483bc840632ce6c0d552f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-22T15:28:21.275 INFO:journalctl@ceph.mon.smithi186.smithi186.stdout:Apr 22 15:28:21 smithi186 bash[74938]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-mon-smithi186 2024-04-22T15:28:21.599 DEBUG:teuthology.orchestra.run.smithi186:> sudo pkill -f 'journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@mon.smithi186.service' 2024-04-22T15:28:21.643 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:28:21.643 INFO:tasks.cephadm.mon.smithi186:Stopped mon.smithi186 2024-04-22T15:28:21.643 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-22T15:28:21.644 DEBUG:teuthology.orchestra.run.smithi119:> sudo systemctl stop ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.0 2024-04-22T15:28:21.945 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:21 smithi119 systemd[1]: Stopping Ceph osd.0 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:28:22.242 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:21 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0[69183]: 2024-04-22T15:28:21.941+0000 7fd986502700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:28:22.242 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:21 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0[69183]: 2024-04-22T15:28:21.941+0000 7fd986502700 -1 osd.0 159 *** Got signal Terminated *** 2024-04-22T15:28:22.242 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:21 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0[69183]: 2024-04-22T15:28:21.941+0000 7fd986502700 -1 osd.0 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:28:27.786 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:27 smithi119 podman[109320]: 2024-04-22 15:28:27.518085882 +0000 UTC m=+5.686737184 container died b35da6e9f2fab056c52fa0b6ba1ee00d77f28e5d0d66e8ebd6de429778a1300f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0, GIT_CLEAN=True, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-22T15:28:28.241 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:27 smithi119 podman[109320]: 2024-04-22 15:28:27.992065232 +0000 UTC m=+6.160716539 container cleanup b35da6e9f2fab056c52fa0b6ba1ee00d77f28e5d0d66e8ebd6de429778a1300f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=) 2024-04-22T15:28:28.241 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:27 smithi119 bash[109320]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0 2024-04-22T15:28:28.617 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:28 smithi119 podman[109338]: 2024-04-22 15:28:28.268596118 +0000 UTC m=+0.747587638 container remove b35da6e9f2fab056c52fa0b6ba1ee00d77f28e5d0d66e8ebd6de429778a1300f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:28:28.901 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:28 smithi119 podman[109431]: 2024-04-22 15:28:28.517549898 +0000 UTC m=+0.018304868 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:28:28.901 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:28 smithi119 podman[109431]: 2024-04-22 15:28:28.655652696 +0000 UTC m=+0.156407682 container create 55e4f50024d96a247b8c8c9e557a51736c6b3793eabe9d750cc890d537df3dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:28:29.202 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:28 smithi119 podman[109431]: 2024-04-22 15:28:28.89877472 +0000 UTC m=+0.399529662 container init 55e4f50024d96a247b8c8c9e557a51736c6b3793eabe9d750cc890d537df3dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-22T15:28:29.202 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:28 smithi119 podman[109431]: 2024-04-22 15:28:28.902180032 +0000 UTC m=+0.402934983 container start 55e4f50024d96a247b8c8c9e557a51736c6b3793eabe9d750cc890d537df3dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-22T15:28:29.202 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:28 smithi119 podman[109431]: 2024-04-22 15:28:28.989625192 +0000 UTC m=+0.490380149 container attach 55e4f50024d96a247b8c8c9e557a51736c6b3793eabe9d750cc890d537df3dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:28:29.491 INFO:journalctl@ceph.osd.0.smithi119.stdout:Apr 22 15:28:29 smithi119 podman[109431]: 2024-04-22 15:28:29.199868256 +0000 UTC m=+0.700623207 container died 55e4f50024d96a247b8c8c9e557a51736c6b3793eabe9d750cc890d537df3dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5) 2024-04-22T15:28:29.899 DEBUG:teuthology.orchestra.run.smithi119:> sudo pkill -f 'journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.0.service' 2024-04-22T15:28:29.937 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:28:29.937 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-22T15:28:29.937 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-22T15:28:29.937 DEBUG:teuthology.orchestra.run.smithi119:> sudo systemctl stop ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.1 2024-04-22T15:28:30.267 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:30 smithi119 systemd[1]: Stopping Ceph osd.1 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:28:30.741 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:30 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[71857]: 2024-04-22T15:28:30.263+0000 7f428aa68700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:28:30.741 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:30 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[71857]: 2024-04-22T15:28:30.263+0000 7f428aa68700 -1 osd.1 159 *** Got signal Terminated *** 2024-04-22T15:28:30.741 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:30 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1[71857]: 2024-04-22T15:28:30.263+0000 7f428aa68700 -1 osd.1 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:28:36.117 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:35 smithi119 podman[109537]: 2024-04-22 15:28:35.814317944 +0000 UTC m=+5.659323483 container died 8e40a47e7598a67892f3da9da47c86367a74df70513deccf39285472d2621ec6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:28:36.442 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:36 smithi119 podman[109537]: 2024-04-22 15:28:36.222182886 +0000 UTC m=+6.067188431 container cleanup 8e40a47e7598a67892f3da9da47c86367a74df70513deccf39285472d2621ec6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS) 2024-04-22T15:28:36.442 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:36 smithi119 bash[109537]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1 2024-04-22T15:28:36.741 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:36 smithi119 podman[109554]: 2024-04-22 15:28:36.440064219 +0000 UTC m=+0.622146177 container remove 8e40a47e7598a67892f3da9da47c86367a74df70513deccf39285472d2621ec6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:28:37.013 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:36 smithi119 podman[109646]: 2024-04-22 15:28:36.662872345 +0000 UTC m=+0.018391265 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:28:37.013 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:36 smithi119 podman[109646]: 2024-04-22 15:28:36.76601464 +0000 UTC m=+0.121533549 container create 00aefee1eae24b9f1089d247cb33cc66ffe6ab5dc91e3e68336f5846282b8847 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-22T15:28:37.013 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:37 smithi119 podman[109646]: 2024-04-22 15:28:37.010649151 +0000 UTC m=+0.366168060 container init 00aefee1eae24b9f1089d247cb33cc66ffe6ab5dc91e3e68336f5846282b8847 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-22T15:28:37.310 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:37 smithi119 podman[109646]: 2024-04-22 15:28:37.013892031 +0000 UTC m=+0.369410941 container start 00aefee1eae24b9f1089d247cb33cc66ffe6ab5dc91e3e68336f5846282b8847 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True) 2024-04-22T15:28:37.310 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:37 smithi119 podman[109646]: 2024-04-22 15:28:37.102332238 +0000 UTC m=+0.457851165 container attach 00aefee1eae24b9f1089d247cb33cc66ffe6ab5dc91e3e68336f5846282b8847 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:28:37.599 INFO:journalctl@ceph.osd.1.smithi119.stdout:Apr 22 15:28:37 smithi119 podman[109646]: 2024-04-22 15:28:37.308234376 +0000 UTC m=+0.663753279 container died 00aefee1eae24b9f1089d247cb33cc66ffe6ab5dc91e3e68336f5846282b8847 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416) 2024-04-22T15:28:37.910 DEBUG:teuthology.orchestra.run.smithi119:> sudo pkill -f 'journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.1.service' 2024-04-22T15:28:37.948 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:28:37.948 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-22T15:28:37.948 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-22T15:28:37.948 DEBUG:teuthology.orchestra.run.smithi119:> sudo systemctl stop ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.2 2024-04-22T15:28:38.241 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:38 smithi119 systemd[1]: Stopping Ceph osd.2 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:28:38.741 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:38 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2[74475]: 2024-04-22T15:28:38.291+0000 7fd41f3dc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:28:38.741 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:38 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2[74475]: 2024-04-22T15:28:38.291+0000 7fd41f3dc700 -1 osd.2 159 *** Got signal Terminated *** 2024-04-22T15:28:38.741 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:38 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2[74475]: 2024-04-22T15:28:38.291+0000 7fd41f3dc700 -1 osd.2 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:28:44.079 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:43 smithi119 podman[109752]: 2024-04-22 15:28:43.766379677 +0000 UTC m=+5.599678829 container died 14287157a66ebaa3b222a19a6bdb8c18aaead191ccd5fc653ac789fee0bbb343 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-22T15:28:44.488 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:44 smithi119 podman[109752]: 2024-04-22 15:28:44.276083018 +0000 UTC m=+6.109382201 container cleanup 14287157a66ebaa3b222a19a6bdb8c18aaead191ccd5fc653ac789fee0bbb343 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:28:44.488 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:44 smithi119 bash[109752]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2 2024-04-22T15:28:44.741 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:44 smithi119 podman[109801]: 2024-04-22 15:28:44.485943382 +0000 UTC m=+0.721041810 container remove 14287157a66ebaa3b222a19a6bdb8c18aaead191ccd5fc653ac789fee0bbb343 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:28:45.154 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:44 smithi119 podman[109893]: 2024-04-22 15:28:44.799232199 +0000 UTC m=+0.018573927 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:28:45.154 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:44 smithi119 podman[109893]: 2024-04-22 15:28:44.945352445 +0000 UTC m=+0.164694208 container create ce3446bec6e5766630b835ff02bf575aa315c34fdd37b33918455cecf725c80f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-22T15:28:45.491 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:45 smithi119 podman[109893]: 2024-04-22 15:28:45.214676563 +0000 UTC m=+0.434018273 container init ce3446bec6e5766630b835ff02bf575aa315c34fdd37b33918455cecf725c80f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-22T15:28:45.491 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:45 smithi119 podman[109893]: 2024-04-22 15:28:45.21771422 +0000 UTC m=+0.437055935 container start ce3446bec6e5766630b835ff02bf575aa315c34fdd37b33918455cecf725c80f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:28:45.491 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:45 smithi119 podman[109893]: 2024-04-22 15:28:45.289723265 +0000 UTC m=+0.509064977 container attach ce3446bec6e5766630b835ff02bf575aa315c34fdd37b33918455cecf725c80f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-22T15:28:45.803 INFO:journalctl@ceph.osd.2.smithi119.stdout:Apr 22 15:28:45 smithi119 podman[109893]: 2024-04-22 15:28:45.510538561 +0000 UTC m=+0.729880274 container died ce3446bec6e5766630b835ff02bf575aa315c34fdd37b33918455cecf725c80f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416) 2024-04-22T15:28:46.095 DEBUG:teuthology.orchestra.run.smithi119:> sudo pkill -f 'journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.2.service' 2024-04-22T15:28:46.134 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:28:46.134 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-22T15:28:46.134 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-22T15:28:46.134 DEBUG:teuthology.orchestra.run.smithi119:> sudo systemctl stop ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.3 2024-04-22T15:28:46.457 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:46 smithi119 systemd[1]: Stopping Ceph osd.3 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:28:46.741 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:46 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[77086]: 2024-04-22T15:28:46.454+0000 7f5dc1ab6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:28:46.741 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:46 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[77086]: 2024-04-22T15:28:46.454+0000 7f5dc1ab6700 -1 osd.3 159 *** Got signal Terminated *** 2024-04-22T15:28:46.742 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:46 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[77086]: 2024-04-22T15:28:46.454+0000 7f5dc1ab6700 -1 osd.3 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:28:48.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:47 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:47.905+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:28:48.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:48 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:48.436+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:23.064275+0000 front 2024-04-22T15:28:23.064151+0000 (oldest deadline 2024-04-22T15:28:48.264388+0000) 2024-04-22T15:28:49.166 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:48 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:48.857+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:28:49.491 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:49 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[77086]: 2024-04-22T15:28:49.191+0000 7f5dbe0aa700 -1 osd.3 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.998666+0000 front 2024-04-22T15:28:22.998883+0000 (oldest deadline 2024-04-22T15:28:48.898717+0000) 2024-04-22T15:28:49.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:49 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:49.423+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:23.064275+0000 front 2024-04-22T15:28:23.064151+0000 (oldest deadline 2024-04-22T15:28:48.264388+0000) 2024-04-22T15:28:49.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:49 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:49.577+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:28:50.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:49 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:49.894+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:28:50.491 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:50 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[77086]: 2024-04-22T15:28:50.223+0000 7f5dbe0aa700 -1 osd.3 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.998666+0000 front 2024-04-22T15:28:22.998883+0000 (oldest deadline 2024-04-22T15:28:48.898717+0000) 2024-04-22T15:28:50.592 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:50 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:50.374+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:23.064275+0000 front 2024-04-22T15:28:23.064151+0000 (oldest deadline 2024-04-22T15:28:48.264388+0000) 2024-04-22T15:28:50.916 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:50 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:50.589+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:28:50.916 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:50 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:50.798+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:28:51.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:50 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:50.913+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:28:51.491 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:51 smithi119 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3[77086]: 2024-04-22T15:28:51.234+0000 7f5dbe0aa700 -1 osd.3 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.998666+0000 front 2024-04-22T15:28:22.998883+0000 (oldest deadline 2024-04-22T15:28:48.898717+0000) 2024-04-22T15:28:51.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:51 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:51.405+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:23.064275+0000 front 2024-04-22T15:28:23.064151+0000 (oldest deadline 2024-04-22T15:28:48.264388+0000) 2024-04-22T15:28:51.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:51 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:51.612+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:28:52.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:51 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:51.869+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:28:52.275 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:52 smithi119 podman[109998]: 2024-04-22 15:28:52.001624553 +0000 UTC m=+5.648060457 container died 2378efa46eca2bdca3f039224d119f028d03df30828466aa811fd0e580497f97 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-22T15:28:52.276 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:51 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:51.788+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:28:52.624 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:52 smithi119 podman[109998]: 2024-04-22 15:28:52.354021292 +0000 UTC m=+6.000457203 container cleanup 2378efa46eca2bdca3f039224d119f028d03df30828466aa811fd0e580497f97 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416) 2024-04-22T15:28:52.624 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:52 smithi119 bash[109998]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3 2024-04-22T15:28:52.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:52 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:52.573+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:28:52.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:52 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:52.420+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:23.064275+0000 front 2024-04-22T15:28:23.064151+0000 (oldest deadline 2024-04-22T15:28:48.264388+0000) 2024-04-22T15:28:52.977 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:52 smithi119 podman[110018]: 2024-04-22 15:28:52.622046786 +0000 UTC m=+0.616137600 container remove 2378efa46eca2bdca3f039224d119f028d03df30828466aa811fd0e580497f97 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-22T15:28:53.241 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:52 smithi119 podman[110112]: 2024-04-22 15:28:52.878076877 +0000 UTC m=+0.018270935 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:28:53.241 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:53 smithi119 podman[110112]: 2024-04-22 15:28:53.014824658 +0000 UTC m=+0.155018746 container create 4a841abdd33b3a540340feb8bcb0a446512a7d511f621a4185f1fc1bd990b1cd (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:28:53.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:52 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:52.840+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:28:53.275 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:52 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:52.787+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:28:53.603 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:53 smithi119 podman[110112]: 2024-04-22 15:28:53.300931626 +0000 UTC m=+0.441125671 container init 4a841abdd33b3a540340feb8bcb0a446512a7d511f621a4185f1fc1bd990b1cd (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-22T15:28:53.604 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:53 smithi119 podman[110112]: 2024-04-22 15:28:53.304540525 +0000 UTC m=+0.444734570 container start 4a841abdd33b3a540340feb8bcb0a446512a7d511f621a4185f1fc1bd990b1cd (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-22T15:28:53.604 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:53 smithi119 podman[110112]: 2024-04-22 15:28:53.384271724 +0000 UTC m=+0.524465793 container attach 4a841abdd33b3a540340feb8bcb0a446512a7d511f621a4185f1fc1bd990b1cd (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:28:53.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:53 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:53.548+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:28:53.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:53 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:53.457+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:23.064275+0000 front 2024-04-22T15:28:23.064151+0000 (oldest deadline 2024-04-22T15:28:48.264388+0000) 2024-04-22T15:28:53.889 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:53 smithi119 podman[110112]: 2024-04-22 15:28:53.601755429 +0000 UTC m=+0.741949474 container died 4a841abdd33b3a540340feb8bcb0a446512a7d511f621a4185f1fc1bd990b1cd (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240416, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-22T15:28:54.140 INFO:journalctl@ceph.osd.3.smithi119.stdout:Apr 22 15:28:54 smithi119 podman[110138]: 2024-04-22 15:28:54.11535038 +0000 UTC m=+0.508980211 container remove 4a841abdd33b3a540340feb8bcb0a446512a7d511f621a4185f1fc1bd990b1cd (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416) 2024-04-22T15:28:54.143 DEBUG:teuthology.orchestra.run.smithi119:> sudo pkill -f 'journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.3.service' 2024-04-22T15:28:54.180 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:28:54.180 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-22T15:28:54.181 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-22T15:28:54.181 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.4 2024-04-22T15:28:54.208 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:53 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:53.850+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:28:54.208 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:53 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:53.813+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:28:54.484 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:54 smithi186 systemd[1]: Stopping Ceph osd.4 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:28:54.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:54 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:54.524+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:28:54.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:54 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:54.480+0000 7f03ef9e5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:28:54.776 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:54 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:54.480+0000 7f03ef9e5700 -1 osd.4 159 *** Got signal Terminated *** 2024-04-22T15:28:54.776 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:54 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:54.480+0000 7f03ef9e5700 -1 osd.4 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:28:54.776 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:54 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:54.503+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:23.064275+0000 front 2024-04-22T15:28:23.064151+0000 (oldest deadline 2024-04-22T15:28:48.264388+0000) 2024-04-22T15:28:55.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:54 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:54.858+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:28:55.275 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:54 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:54.818+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:28:55.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:55 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:55.485+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:23.064275+0000 front 2024-04-22T15:28:23.064151+0000 (oldest deadline 2024-04-22T15:28:48.264388+0000) 2024-04-22T15:28:55.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:55 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:55.560+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:28:56.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:55 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:55.872+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:28:56.275 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:55 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:55.819+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:28:56.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:56 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:56.518+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:28:56.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:56 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:56.444+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:23.064275+0000 front 2024-04-22T15:28:23.064151+0000 (oldest deadline 2024-04-22T15:28:48.264388+0000) 2024-04-22T15:28:57.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:56 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:56.829+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:28:57.275 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:56 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:56.795+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:28:57.752 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:57 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:57.451+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:23.064275+0000 front 2024-04-22T15:28:23.064151+0000 (oldest deadline 2024-04-22T15:28:48.264388+0000) 2024-04-22T15:28:57.752 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:57 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:57.502+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:28:58.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:57 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:57.835+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:28:58.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:57 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:57.749+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:28:58.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:57 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:57.749+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:28:58.761 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:58.461+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:23.064275+0000 front 2024-04-22T15:28:23.064151+0000 (oldest deadline 2024-04-22T15:28:48.264388+0000) 2024-04-22T15:28:58.761 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:58.461+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:28:58.761 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:58.461+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:34.235388+0000 front 2024-04-22T15:28:34.235429+0000 (oldest deadline 2024-04-22T15:28:57.735460+0000) 2024-04-22T15:28:59.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:58.758+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:28:59.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:58.758+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:28:59.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:58.846+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:28:59.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:58 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:58.846+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:28:59.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:59 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:59.468+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:23.064275+0000 front 2024-04-22T15:28:23.064151+0000 (oldest deadline 2024-04-22T15:28:48.264388+0000) 2024-04-22T15:28:59.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:59 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4[54792]: 2024-04-22T15:28:59.468+0000 7f03e7fd1700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:34.165557+0000 front 2024-04-22T15:28:34.165449+0000 (oldest deadline 2024-04-22T15:28:58.865415+0000) 2024-04-22T15:28:59.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:59 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:59.455+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:28:59.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:28:59 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:28:59.455+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:34.235388+0000 front 2024-04-22T15:28:34.235429+0000 (oldest deadline 2024-04-22T15:28:57.735460+0000) 2024-04-22T15:29:00.192 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:28:59 smithi186 podman[75104]: 2024-04-22 15:28:59.863458491 +0000 UTC m=+5.491137618 container died 9518e2eb7665dff135d727a54ea6487618b412d6d483a88924e4e60c2c2d92b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_CLEAN=True, ceph=True) 2024-04-22T15:29:00.193 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:59 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:59.811+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:00.193 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:28:59 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:28:59.811+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:00.193 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:59 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:59.787+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:00.193 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:28:59 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:28:59.787+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:00.525 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:29:00 smithi186 podman[75104]: 2024-04-22 15:29:00.389470346 +0000 UTC m=+6.017149461 container cleanup 9518e2eb7665dff135d727a54ea6487618b412d6d483a88924e4e60c2c2d92b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, ceph=True, CEPH_POINT_RELEASE=) 2024-04-22T15:29:00.525 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:29:00 smithi186 bash[75104]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4 2024-04-22T15:29:00.525 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:00 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:00.404+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:29:00.525 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:00 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:00.404+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:34.235388+0000 front 2024-04-22T15:28:34.235429+0000 (oldest deadline 2024-04-22T15:28:57.735460+0000) 2024-04-22T15:29:01.001 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:29:00 smithi186 podman[75123]: 2024-04-22 15:29:00.591097591 +0000 UTC m=+0.730018249 container remove 9518e2eb7665dff135d727a54ea6487618b412d6d483a88924e4e60c2c2d92b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-22T15:29:01.002 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:00 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:00.812+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:01.002 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:00 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:00.813+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:01.002 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:00 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:00.820+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:01.002 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:00 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:00.820+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:01.275 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:29:00 smithi186 podman[75249]: 2024-04-22 15:29:00.902260186 +0000 UTC m=+0.018091813 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:29:01.275 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:29:01 smithi186 podman[75249]: 2024-04-22 15:29:01.000108501 +0000 UTC m=+0.115940139 container create 45abdbc038820d04121b86b7603acb8ecbde3eb2f244a19472beedbebfd4d4ed (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.build-date=20240416) 2024-04-22T15:29:01.275 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:29:01 smithi186 podman[75249]: 2024-04-22 15:29:01.210922659 +0000 UTC m=+0.326754286 container init 45abdbc038820d04121b86b7603acb8ecbde3eb2f244a19472beedbebfd4d4ed (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-22T15:29:01.275 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:29:01 smithi186 podman[75249]: 2024-04-22 15:29:01.214061845 +0000 UTC m=+0.329893473 container start 45abdbc038820d04121b86b7603acb8ecbde3eb2f244a19472beedbebfd4d4ed (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-22T15:29:01.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:29:01 smithi186 podman[75249]: 2024-04-22 15:29:01.319912789 +0000 UTC m=+0.435744433 container attach 45abdbc038820d04121b86b7603acb8ecbde3eb2f244a19472beedbebfd4d4ed (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:29:01.775 INFO:journalctl@ceph.osd.4.smithi186.stdout:Apr 22 15:29:01 smithi186 podman[75249]: 2024-04-22 15:29:01.521289626 +0000 UTC m=+0.637121246 container died 45abdbc038820d04121b86b7603acb8ecbde3eb2f244a19472beedbebfd4d4ed (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:29:01.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:01 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:01.434+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:29:01.776 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:01 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:01.434+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:34.235388+0000 front 2024-04-22T15:28:34.235429+0000 (oldest deadline 2024-04-22T15:28:57.735460+0000) 2024-04-22T15:29:02.142 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:01 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:01.810+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:02.142 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:01 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:01.810+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:02.142 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:01 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:01.794+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:02.143 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:01 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:01.794+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:02.166 DEBUG:teuthology.orchestra.run.smithi186:> sudo pkill -f 'journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.4.service' 2024-04-22T15:29:02.247 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:29:02.247 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-22T15:29:02.247 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-22T15:29:02.247 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.5 2024-04-22T15:29:02.453 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:02 smithi186 systemd[1]: Stopping Ceph osd.5 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:29:02.763 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:02 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:02.448+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:29:02.763 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:02 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:02.448+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:34.235388+0000 front 2024-04-22T15:28:34.235429+0000 (oldest deadline 2024-04-22T15:28:57.735460+0000) 2024-04-22T15:29:02.763 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:02 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:02.549+0000 7f018a899700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:29:02.763 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:02 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:02.549+0000 7f018a899700 -1 osd.5 159 *** Got signal Terminated *** 2024-04-22T15:29:02.763 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:02 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:02.549+0000 7f018a899700 -1 osd.5 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:29:03.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:02 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:02.824+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:03.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:02 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:02.824+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:03.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:02 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:02.759+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:03.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:02 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:02.759+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:03.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:03 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:03.416+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:29:03.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:03 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:03.416+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:34.235388+0000 front 2024-04-22T15:28:34.235429+0000 (oldest deadline 2024-04-22T15:28:57.735460+0000) 2024-04-22T15:29:04.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:03 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:03.799+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:04.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:03 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:03.799+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:04.275 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:03 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:03.773+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:04.275 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:03 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:03.773+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:04.741 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:04 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:04.458+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:29:04.742 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:04 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:04.458+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:34.235388+0000 front 2024-04-22T15:28:34.235429+0000 (oldest deadline 2024-04-22T15:28:57.735460+0000) 2024-04-22T15:29:05.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:04 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:04.809+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:05.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:04 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:04.809+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:05.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:04 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:04.809+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.584438+0000 front 2024-04-22T15:28:41.584505+0000 (oldest deadline 2024-04-22T15:29:03.884249+0000) 2024-04-22T15:29:05.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:04 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:04.738+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:05.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:04 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:04.738+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:05.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:05 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:05.415+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:29:05.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:05 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:05.415+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:34.235388+0000 front 2024-04-22T15:28:34.235429+0000 (oldest deadline 2024-04-22T15:28:57.735460+0000) 2024-04-22T15:29:06.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:05 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:05.795+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:06.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:05 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:05.795+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:06.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:05 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:05.795+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.584438+0000 front 2024-04-22T15:28:41.584505+0000 (oldest deadline 2024-04-22T15:29:03.884249+0000) 2024-04-22T15:29:06.275 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:05 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:05.786+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:06.275 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:05 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:05.786+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:06.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:06 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:06.417+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:29:06.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:06 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:06.417+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:34.235388+0000 front 2024-04-22T15:28:34.235429+0000 (oldest deadline 2024-04-22T15:28:57.735460+0000) 2024-04-22T15:29:07.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:06 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:06.838+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:07.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:06 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:06.838+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:07.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:06 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:06.838+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.584438+0000 front 2024-04-22T15:28:41.584505+0000 (oldest deadline 2024-04-22T15:29:03.884249+0000) 2024-04-22T15:29:07.275 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:06 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:06.814+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:07.275 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:06 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:06.814+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:07.275 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:06 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:06.814+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:07.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:07 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:07.431+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:24.834474+0000 front 2024-04-22T15:28:24.834263+0000 (oldest deadline 2024-04-22T15:28:49.534271+0000) 2024-04-22T15:29:07.775 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:07 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5[57083]: 2024-04-22T15:29:07.431+0000 7f0186e8d700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:34.235388+0000 front 2024-04-22T15:28:34.235429+0000 (oldest deadline 2024-04-22T15:28:57.735460+0000) 2024-04-22T15:29:08.059 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:07 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:07.842+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:08.059 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:07 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:07.842+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:08.059 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:07 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:07.842+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.584438+0000 front 2024-04-22T15:28:41.584505+0000 (oldest deadline 2024-04-22T15:29:03.884249+0000) 2024-04-22T15:29:08.059 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:07 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:07.772+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:08.059 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:07 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:07.772+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:08.059 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:07 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:07.772+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:08.312 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:08 smithi186 podman[75354]: 2024-04-22 15:29:08.056453 +0000 UTC m=+5.590570431 container died e1c8f1367a89bcf8496f086a8ba91bcbb76a0341d0017d57a533ab019694069c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD) 2024-04-22T15:29:08.670 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:08 smithi186 podman[75354]: 2024-04-22 15:29:08.475786168 +0000 UTC m=+6.009903595 container cleanup e1c8f1367a89bcf8496f086a8ba91bcbb76a0341d0017d57a533ab019694069c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:29:08.671 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:08 smithi186 bash[75354]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5 2024-04-22T15:29:08.972 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:08 smithi186 podman[75372]: 2024-04-22 15:29:08.668495515 +0000 UTC m=+0.604952931 container remove e1c8f1367a89bcf8496f086a8ba91bcbb76a0341d0017d57a533ab019694069c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-22T15:29:08.972 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:08 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:08.739+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:08.972 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:08 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:08.739+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:08.972 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:08 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:08.739+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:08.972 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:08 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:08.885+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:08.972 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:08 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:08.885+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:08.973 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:08 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:08.885+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.584438+0000 front 2024-04-22T15:28:41.584505+0000 (oldest deadline 2024-04-22T15:29:03.884249+0000) 2024-04-22T15:29:09.249 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:08 smithi186 podman[75462]: 2024-04-22 15:29:08.969633462 +0000 UTC m=+0.099865457 container create de10be411394f43573afd0a69e65faab737d8965b85d918e881c5728adbceea2 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5) 2024-04-22T15:29:09.249 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:08 smithi186 podman[75462]: 2024-04-22 15:29:08.888222705 +0000 UTC m=+0.018454707 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:29:09.526 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:09 smithi186 podman[75462]: 2024-04-22 15:29:09.247290995 +0000 UTC m=+0.377522993 container init de10be411394f43573afd0a69e65faab737d8965b85d918e881c5728adbceea2 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:29:09.526 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:09 smithi186 podman[75462]: 2024-04-22 15:29:09.250773886 +0000 UTC m=+0.381005885 container start de10be411394f43573afd0a69e65faab737d8965b85d918e881c5728adbceea2 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS) 2024-04-22T15:29:09.526 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:09 smithi186 podman[75462]: 2024-04-22 15:29:09.322082754 +0000 UTC m=+0.452314766 container attach de10be411394f43573afd0a69e65faab737d8965b85d918e881c5728adbceea2 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5) 2024-04-22T15:29:09.836 INFO:journalctl@ceph.osd.5.smithi186.stdout:Apr 22 15:29:09 smithi186 podman[75462]: 2024-04-22 15:29:09.542731723 +0000 UTC m=+0.672963723 container died de10be411394f43573afd0a69e65faab737d8965b85d918e881c5728adbceea2 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:29:09.837 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:09 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:09.763+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:09.837 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:09 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:09.763+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:09.837 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:09 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:09.763+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:10.094 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:09 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:09.861+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:10.094 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:09 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:09.861+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:10.094 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:09 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:09.861+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.584438+0000 front 2024-04-22T15:28:41.584505+0000 (oldest deadline 2024-04-22T15:29:03.884249+0000) 2024-04-22T15:29:10.119 DEBUG:teuthology.orchestra.run.smithi186:> sudo pkill -f 'journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.5.service' 2024-04-22T15:29:10.163 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:29:10.163 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-22T15:29:10.163 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-22T15:29:10.163 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.6 2024-04-22T15:29:10.497 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:10 smithi186 systemd[1]: Stopping Ceph osd.6 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:29:10.775 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:10 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:10.494+0000 7fabd6237700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:29:10.775 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:10 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:10.494+0000 7fabd6237700 -1 osd.6 159 *** Got signal Terminated *** 2024-04-22T15:29:10.775 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:10 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:10.494+0000 7fabd6237700 -1 osd.6 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:29:10.775 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:10 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:10.717+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:10.776 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:10 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:10.717+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:10.776 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:10 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:10.717+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:11.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:10 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:10.824+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:11.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:10 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:10.824+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:11.275 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:10 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:10.824+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.584438+0000 front 2024-04-22T15:28:41.584505+0000 (oldest deadline 2024-04-22T15:29:03.884249+0000) 2024-04-22T15:29:12.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:11 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:11.849+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:12.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:11 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:11.849+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:12.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:11 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:11.849+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.584438+0000 front 2024-04-22T15:28:41.584505+0000 (oldest deadline 2024-04-22T15:29:03.884249+0000) 2024-04-22T15:29:12.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:11 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:11.735+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:12.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:11 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:11.735+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:12.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:11 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:11.735+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:13.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:12.833+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:13.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:12.833+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:13.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:12.833+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.584438+0000 front 2024-04-22T15:28:41.584505+0000 (oldest deadline 2024-04-22T15:29:03.884249+0000) 2024-04-22T15:29:13.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:12.754+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:13.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:12.754+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:13.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:12 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:12.754+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:14.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:13 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:13.798+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:14.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:13 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:13.798+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:14.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:13 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:13.798+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.584438+0000 front 2024-04-22T15:28:41.584505+0000 (oldest deadline 2024-04-22T15:29:03.884249+0000) 2024-04-22T15:29:14.026 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:13 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:13.709+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:14.026 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:13 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:13.709+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:14.026 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:13 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:13.709+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:15.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:14 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:14.805+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:22.381427+0000 front 2024-04-22T15:28:22.381518+0000 (oldest deadline 2024-04-22T15:28:47.681388+0000) 2024-04-22T15:29:15.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:14 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:14.805+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:33.482917+0000 front 2024-04-22T15:28:33.482732+0000 (oldest deadline 2024-04-22T15:28:58.182708+0000) 2024-04-22T15:29:15.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:14 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6[59332]: 2024-04-22T15:29:14.805+0000 7fabce823700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.584438+0000 front 2024-04-22T15:28:41.584505+0000 (oldest deadline 2024-04-22T15:29:03.884249+0000) 2024-04-22T15:29:15.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:14 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:14.747+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:15.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:14 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:14.747+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:15.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:14 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:14.747+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:15.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:14 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:14.747+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6830 osd.3 since back 2024-04-22T15:28:49.323234+0000 front 2024-04-22T15:28:49.323194+0000 (oldest deadline 2024-04-22T15:29:14.623149+0000) 2024-04-22T15:29:16.025 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:16 smithi186 podman[75568]: 2024-04-22 15:29:16.002557569 +0000 UTC m=+5.618066412 container died 31e40cdb753c94695394c0a44bc57dd1ae5bafa454f24fcc7ad25cc22f4533e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-22T15:29:16.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:15 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:15.754+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:16.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:15 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:15.754+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:16.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:15 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:15.754+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:16.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:15 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:15.754+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6830 osd.3 since back 2024-04-22T15:28:49.323234+0000 front 2024-04-22T15:28:49.323194+0000 (oldest deadline 2024-04-22T15:29:14.623149+0000) 2024-04-22T15:29:16.623 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:16 smithi186 podman[75568]: 2024-04-22 15:29:16.411488668 +0000 UTC m=+6.026997509 container cleanup 31e40cdb753c94695394c0a44bc57dd1ae5bafa454f24fcc7ad25cc22f4533e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416) 2024-04-22T15:29:16.623 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:16 smithi186 bash[75568]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6 2024-04-22T15:29:16.915 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:16 smithi186 podman[75586]: 2024-04-22 15:29:16.621108425 +0000 UTC m=+0.610524704 container remove 31e40cdb753c94695394c0a44bc57dd1ae5bafa454f24fcc7ad25cc22f4533e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=) 2024-04-22T15:29:16.915 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:16 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:16.753+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:16.915 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:16 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:16.753+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:16.915 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:16 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:16.753+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:16.915 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:16 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:16.753+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6830 osd.3 since back 2024-04-22T15:28:49.323234+0000 front 2024-04-22T15:28:49.323194+0000 (oldest deadline 2024-04-22T15:29:14.623149+0000) 2024-04-22T15:29:17.227 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:16 smithi186 podman[75681]: 2024-04-22 15:29:16.816078619 +0000 UTC m=+0.017705893 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:29:17.227 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:16 smithi186 podman[75681]: 2024-04-22 15:29:16.938531325 +0000 UTC m=+0.140158634 container create 37c694a3d063915dd9036e0714a5468da4899be3e174b63e9a0eb5ff9c70ed62 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True) 2024-04-22T15:29:17.227 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:17 smithi186 podman[75681]: 2024-04-22 15:29:17.157701594 +0000 UTC m=+0.359328857 container init 37c694a3d063915dd9036e0714a5468da4899be3e174b63e9a0eb5ff9c70ed62 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-22T15:29:17.227 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:17 smithi186 podman[75681]: 2024-04-22 15:29:17.160783104 +0000 UTC m=+0.362410367 container start 37c694a3d063915dd9036e0714a5468da4899be3e174b63e9a0eb5ff9c70ed62 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:29:17.227 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:17 smithi186 podman[75681]: 2024-04-22 15:29:17.22417938 +0000 UTC m=+0.425806643 container attach 37c694a3d063915dd9036e0714a5468da4899be3e174b63e9a0eb5ff9c70ed62 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-22T15:29:17.525 INFO:journalctl@ceph.osd.6.smithi186.stdout:Apr 22 15:29:17 smithi186 podman[75681]: 2024-04-22 15:29:17.461557902 +0000 UTC m=+0.663185163 container died 37c694a3d063915dd9036e0714a5468da4899be3e174b63e9a0eb5ff9c70ed62 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:29:18.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:17 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:17.709+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:18.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:17 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:17.709+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:18.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:17 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:17.709+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:18.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:17 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:17.709+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6830 osd.3 since back 2024-04-22T15:28:49.323234+0000 front 2024-04-22T15:28:49.323194+0000 (oldest deadline 2024-04-22T15:29:14.623149+0000) 2024-04-22T15:29:18.186 DEBUG:teuthology.orchestra.run.smithi186:> sudo pkill -f 'journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.6.service' 2024-04-22T15:29:18.224 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:29:18.224 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-22T15:29:18.224 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-22T15:29:18.224 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.7 2024-04-22T15:29:18.566 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:18 smithi186 systemd[1]: Stopping Ceph osd.7 for 74739db6-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:29:19.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:18 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:18.563+0000 7f6d56cb5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:29:19.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:18 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:18.563+0000 7f6d56cb5700 -1 osd.7 159 *** Got signal Terminated *** 2024-04-22T15:29:19.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:18 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:18.563+0000 7f6d56cb5700 -1 osd.7 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:29:19.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:18 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:18.680+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:19.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:18 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:18.680+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:19.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:18 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:18.680+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:19.026 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:18 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:18.680+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6830 osd.3 since back 2024-04-22T15:28:49.323234+0000 front 2024-04-22T15:28:49.323194+0000 (oldest deadline 2024-04-22T15:29:14.623149+0000) 2024-04-22T15:29:20.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:19 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:19.703+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:20.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:19 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:19.704+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:20.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:19 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:19.704+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:20.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:19 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:19.704+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6830 osd.3 since back 2024-04-22T15:28:49.323234+0000 front 2024-04-22T15:28:49.323194+0000 (oldest deadline 2024-04-22T15:29:14.623149+0000) 2024-04-22T15:29:20.775 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:20 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:20.657+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:20.775 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:20 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:20.657+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:20.775 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:20 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:20.657+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:20.775 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:20 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:20.657+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6830 osd.3 since back 2024-04-22T15:28:49.323234+0000 front 2024-04-22T15:28:49.323194+0000 (oldest deadline 2024-04-22T15:29:14.623149+0000) 2024-04-22T15:29:20.775 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:20 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:20.657+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.186:6804 osd.4 since back 2024-04-22T15:28:58.724100+0000 front 2024-04-22T15:28:58.724004+0000 (oldest deadline 2024-04-22T15:29:20.423959+0000) 2024-04-22T15:29:22.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:21.707+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:22.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:21.707+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:22.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:21.707+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:22.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:21.707+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6830 osd.3 since back 2024-04-22T15:28:49.323234+0000 front 2024-04-22T15:28:49.323194+0000 (oldest deadline 2024-04-22T15:29:14.623149+0000) 2024-04-22T15:29:22.026 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:21 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:21.707+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.186:6804 osd.4 since back 2024-04-22T15:28:58.724100+0000 front 2024-04-22T15:28:58.724004+0000 (oldest deadline 2024-04-22T15:29:20.423959+0000) 2024-04-22T15:29:23.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:22 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:22.752+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6806 osd.0 since back 2024-04-22T15:28:25.718954+0000 front 2024-04-22T15:28:25.719072+0000 (oldest deadline 2024-04-22T15:28:50.418830+0000) 2024-04-22T15:29:23.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:22 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:22.752+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6814 osd.1 since back 2024-04-22T15:28:35.020306+0000 front 2024-04-22T15:28:35.020283+0000 (oldest deadline 2024-04-22T15:28:57.320322+0000) 2024-04-22T15:29:23.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:22 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:22.752+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6822 osd.2 since back 2024-04-22T15:28:41.822085+0000 front 2024-04-22T15:28:41.822198+0000 (oldest deadline 2024-04-22T15:29:06.522019+0000) 2024-04-22T15:29:23.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:22 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:22.752+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.119:6830 osd.3 since back 2024-04-22T15:28:49.323234+0000 front 2024-04-22T15:28:49.323194+0000 (oldest deadline 2024-04-22T15:29:14.623149+0000) 2024-04-22T15:29:23.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:22 smithi186 ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7[61592]: 2024-04-22T15:29:22.752+0000 7f6d4f2a1700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.186:6804 osd.4 since back 2024-04-22T15:28:58.724100+0000 front 2024-04-22T15:28:58.724004+0000 (oldest deadline 2024-04-22T15:29:20.423959+0000) 2024-04-22T15:29:24.292 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:24 smithi186 podman[75786]: 2024-04-22 15:29:24.02329294 +0000 UTC m=+5.583395816 container died 2f6a10f5d68af3ce971206a5eb05cb4babf3e1b2ebe05907fa6fb639d29d7aae (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:29:24.659 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:24 smithi186 podman[75786]: 2024-04-22 15:29:24.455452859 +0000 UTC m=+6.015555730 container cleanup 2f6a10f5d68af3ce971206a5eb05cb4babf3e1b2ebe05907fa6fb639d29d7aae (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-22T15:29:24.659 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:24 smithi186 bash[75786]: ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7 2024-04-22T15:29:25.025 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:24 smithi186 podman[75847]: 2024-04-22 15:29:24.656679055 +0000 UTC m=+0.629651123 container remove 2f6a10f5d68af3ce971206a5eb05cb4babf3e1b2ebe05907fa6fb639d29d7aae (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-22T15:29:25.404 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:25 smithi186 podman[75939]: 2024-04-22 15:29:24.977355765 +0000 UTC m=+0.018304901 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:29:25.404 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:25 smithi186 podman[75939]: 2024-04-22 15:29:25.091009925 +0000 UTC m=+0.131959079 container create e0ec36231f12087df1ca39c28073ecf1622fb455f1b4be891abdcc32cc934f5c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240416, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-22T15:29:25.404 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:25 smithi186 podman[75939]: 2024-04-22 15:29:25.318597797 +0000 UTC m=+0.359546908 container init e0ec36231f12087df1ca39c28073ecf1622fb455f1b4be891abdcc32cc934f5c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-22T15:29:25.404 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:25 smithi186 podman[75939]: 2024-04-22 15:29:25.321632456 +0000 UTC m=+0.362581571 container start e0ec36231f12087df1ca39c28073ecf1622fb455f1b4be891abdcc32cc934f5c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-22T15:29:25.404 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:25 smithi186 podman[75939]: 2024-04-22 15:29:25.4016517 +0000 UTC m=+0.442600831 container attach e0ec36231f12087df1ca39c28073ecf1622fb455f1b4be891abdcc32cc934f5c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-22T15:29:25.775 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:25 smithi186 podman[75939]: 2024-04-22 15:29:25.614596813 +0000 UTC m=+0.655545946 container died e0ec36231f12087df1ca39c28073ecf1622fb455f1b4be891abdcc32cc934f5c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-22T15:29:26.157 INFO:journalctl@ceph.osd.7.smithi186.stdout:Apr 22 15:29:26 smithi186 podman[75965]: 2024-04-22 15:29:26.137982089 +0000 UTC m=+0.518072359 container remove e0ec36231f12087df1ca39c28073ecf1622fb455f1b4be891abdcc32cc934f5c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-74739db6-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux ) 2024-04-22T15:29:26.162 DEBUG:teuthology.orchestra.run.smithi186:> sudo pkill -f 'journalctl -f -n 0 -u ceph-74739db6-00b9-11ef-bc93-c7b262605968@osd.7.service' 2024-04-22T15:29:26.198 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:29:26.198 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-22T15:29:26.199 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 74739db6-00b9-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-22T15:32:23.725 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 74739db6-00b9-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-22T15:32:44.857 DEBUG:teuthology.orchestra.run.smithi119:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:32:44.890 DEBUG:teuthology.orchestra.run.smithi186:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:32:44.919 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-22T15:32:44.921 DEBUG:teuthology.misc:Transferring archived files from smithi119:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668325/remote/smithi119/crash 2024-04-22T15:32:44.922 DEBUG:teuthology.orchestra.run.smithi119:> sudo tar c -f - -C /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/crash -- . 2024-04-22T15:32:45.003 INFO:teuthology.orchestra.run.smithi119.stderr:tar: /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-22T15:32:45.004 INFO:teuthology.orchestra.run.smithi119.stderr:tar: Error is not recoverable: exiting now 2024-04-22T15:32:45.005 DEBUG:teuthology.misc:Transferring archived files from smithi186:/var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668325/remote/smithi186/crash 2024-04-22T15:32:45.006 DEBUG:teuthology.orchestra.run.smithi186:> sudo tar c -f - -C /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/crash -- . 2024-04-22T15:32:45.038 INFO:teuthology.orchestra.run.smithi186.stderr:tar: /var/lib/ceph/74739db6-00b9-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-22T15:32:45.038 INFO:teuthology.orchestra.run.smithi186.stderr:tar: Error is not recoverable: exiting now 2024-04-22T15:32:45.039 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-22T15:32:45.039 DEBUG:teuthology.orchestra.run.smithi119:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-22T15:32:45.068 INFO:tasks.cephadm:Compressing logs... 2024-04-22T15:32:45.068 DEBUG:teuthology.orchestra.run.smithi119:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-22T15:32:45.110 DEBUG:teuthology.orchestra.run.smithi186:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-22T15:32:45.135 INFO:teuthology.orchestra.run.smithi119.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.log 2024-04-22T15:32:45.135 INFO:teuthology.orchestra.run.smithi119.stderr:: No such file or directory 2024-04-22T15:32:45.135 INFO:teuthology.orchestra.run.smithi119.stderr:gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-22T15:32:45.136 INFO:teuthology.orchestra.run.smithi119.stderr:gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-22T15:32:45.136 INFO:teuthology.orchestra.run.smithi119.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.log: /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-22T15:32:45.136 INFO:teuthology.orchestra.run.smithi119.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-22T15:32:45.136 INFO:teuthology.orchestra.run.smithi119.stderr:gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-22T15:32:45.136 INFO:teuthology.orchestra.run.smithi119.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.3.log: /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-22T15:32:45.136 INFO:teuthology.orchestra.run.smithi119.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mon.smithi119.log 2024-04-22T15:32:45.136 INFO:teuthology.orchestra.run.smithi119.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mgr.smithi119.jjbxxd.log 2024-04-22T15:32:45.140 INFO:teuthology.orchestra.run.smithi119.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mon.smithi119.log: 85.8% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-22T15:32:45.140 INFO:teuthology.orchestra.run.smithi119.stderr:gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-22T15:32:45.140 INFO:teuthology.orchestra.run.smithi119.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mgr.smithi119.jjbxxd.log: 87.1% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-22T15:32:45.140 INFO:teuthology.orchestra.run.smithi119.stderr:gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi119.ncbqaf.log 2024-04-22T15:32:45.140 INFO:teuthology.orchestra.run.smithi119.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-volume.log: 91.6% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-22T15:32:45.141 INFO:teuthology.orchestra.run.smithi119.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-22T15:32:45.145 INFO:teuthology.orchestra.run.smithi186.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.log 2024-04-22T15:32:45.145 INFO:teuthology.orchestra.run.smithi186.stderr:: No such file or directory 2024-04-22T15:32:45.145 INFO:teuthology.orchestra.run.smithi186.stderr:gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-22T15:32:45.145 INFO:teuthology.orchestra.run.smithi186.stderr:gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-22T15:32:45.145 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-22T15:32:45.145 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.audit.log: /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-22T15:32:45.145 INFO:teuthology.orchestra.run.smithi186.stderr:gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-22T15:32:45.145 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.5.log: /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi186.dglszf.log 2024-04-22T15:32:45.146 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-22T15:32:45.146 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi186.dglszf.log: gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mon.smithi186.log 2024-04-22T15:32:45.146 INFO:teuthology.orchestra.run.smithi186.stderr: 88.5% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi186.dglszf.log.gz 2024-04-22T15:32:45.146 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.cephadm.log: 85.8% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-22T15:32:45.146 INFO:teuthology.orchestra.run.smithi186.stderr:gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mgr.smithi186.vtjgwg.log 2024-04-22T15:32:45.146 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mon.smithi186.log: 87.1% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-22T15:32:45.146 INFO:teuthology.orchestra.run.smithi186.stderr:gzip -5 --verbose -- /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-22T15:32:45.146 INFO:teuthology.orchestra.run.smithi186.stderr: 91.7%gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-22T15:32:45.147 INFO:teuthology.orchestra.run.smithi186.stderr: -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-22T15:32:45.148 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mgr.smithi186.vtjgwg.log: /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-22T15:32:45.153 INFO:teuthology.orchestra.run.smithi186.stderr: 91.2% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-22T15:32:45.155 INFO:teuthology.orchestra.run.smithi119.stderr:/var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi119.ncbqaf.log: /var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-22T15:32:45.164 INFO:teuthology.orchestra.run.smithi119.stderr: 91.4% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi119.ncbqaf.log.gz 2024-04-22T15:32:45.186 INFO:teuthology.orchestra.run.smithi119.stderr: 91.6% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-22T15:32:45.187 INFO:teuthology.orchestra.run.smithi186.stderr: 89.7% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mgr.smithi186.vtjgwg.log.gz 2024-04-22T15:32:45.494 INFO:teuthology.orchestra.run.smithi186.stderr: 92.7% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mon.smithi186.log.gz 2024-04-22T15:32:45.545 INFO:teuthology.orchestra.run.smithi119.stderr: 88.4% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mgr.smithi119.jjbxxd.log.gz 2024-04-22T15:32:46.484 INFO:teuthology.orchestra.run.smithi119.stderr: 92.0% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-mon.smithi119.log.gz 2024-04-22T15:32:58.160 INFO:teuthology.orchestra.run.smithi119.stderr: 94.5% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-22T15:33:00.888 INFO:teuthology.orchestra.run.smithi186.stderr: 93.9% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-22T15:33:03.085 INFO:teuthology.orchestra.run.smithi119.stderr: 94.1% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-22T15:33:03.966 INFO:teuthology.orchestra.run.smithi186.stderr: 94.3% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-22T15:33:04.231 INFO:teuthology.orchestra.run.smithi119.stderr: 94.2% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-22T15:33:05.604 INFO:teuthology.orchestra.run.smithi186.stderr: 94.2% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-22T15:33:06.778 INFO:teuthology.orchestra.run.smithi119.stderr: 94.1% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-22T15:33:06.779 INFO:teuthology.orchestra.run.smithi119.stderr: 2024-04-22T15:33:06.779 INFO:teuthology.orchestra.run.smithi119.stderr:real 0m21.657s 2024-04-22T15:33:06.779 INFO:teuthology.orchestra.run.smithi119.stderr:user 1m10.742s 2024-04-22T15:33:06.779 INFO:teuthology.orchestra.run.smithi119.stderr:sys 0m2.696s 2024-04-22T15:33:11.188 INFO:teuthology.orchestra.run.smithi186.stderr: 94.1% -- replaced with /var/log/ceph/74739db6-00b9-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-22T15:33:11.190 INFO:teuthology.orchestra.run.smithi186.stderr: 2024-04-22T15:33:11.190 INFO:teuthology.orchestra.run.smithi186.stderr:real 0m26.065s 2024-04-22T15:33:11.190 INFO:teuthology.orchestra.run.smithi186.stderr:user 1m18.412s 2024-04-22T15:33:11.190 INFO:teuthology.orchestra.run.smithi186.stderr:sys 0m2.995s 2024-04-22T15:33:11.190 INFO:tasks.cephadm:Archiving logs... 2024-04-22T15:33:11.191 DEBUG:teuthology.misc:Transferring archived files from smithi119:/var/log/ceph to /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668325/remote/smithi119/log 2024-04-22T15:33:11.192 DEBUG:teuthology.orchestra.run.smithi119:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-22T15:33:21.167 DEBUG:teuthology.misc:Transferring archived files from smithi186:/var/log/ceph to /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668325/remote/smithi186/log 2024-04-22T15:33:21.168 DEBUG:teuthology.orchestra.run.smithi186:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-22T15:33:32.541 INFO:tasks.cephadm:Removing cluster... 2024-04-22T15:33:32.542 DEBUG:teuthology.orchestra.run.smithi119:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 74739db6-00b9-11ef-bc93-c7b262605968 --force 2024-04-22T15:33:32.912 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 74739db6-00b9-11ef-bc93-c7b262605968 --force 2024-04-22T15:33:33.302 INFO:tasks.cephadm:Removing cephadm ... 2024-04-22T15:33:33.303 DEBUG:teuthology.orchestra.run.smithi119:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-22T15:33:33.318 DEBUG:teuthology.orchestra.run.smithi186:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-22T15:33:33.333 INFO:tasks.cephadm:Teardown complete 2024-04-22T15:33:33.333 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-22T15:33:33.344 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-22T15:33:33.344 DEBUG:teuthology.orchestra.run.smithi119:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T15:33:33.361 DEBUG:teuthology.orchestra.run.smithi186:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T15:33:33.373 INFO:teuthology.orchestra.run.smithi119.stderr:bash: line 1: ntpq: command not found 2024-04-22T15:33:33.378 INFO:teuthology.orchestra.run.smithi119.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T15:33:33.378 INFO:teuthology.orchestra.run.smithi119.stdout:=============================================================================== 2024-04-22T15:33:33.378 INFO:teuthology.orchestra.run.smithi119.stdout:^- hv01.front.sepia.ceph.com 2 6 377 59 -1939us[-1875us] +/- 570ms 2024-04-22T15:33:33.378 INFO:teuthology.orchestra.run.smithi119.stdout:^* hv02.front.sepia.ceph.com 3 6 377 59 -567us[ -504us] +/- 54ms 2024-04-22T15:33:33.378 INFO:teuthology.orchestra.run.smithi119.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 60 +710us[ +774us] +/- 65ms 2024-04-22T15:33:33.378 INFO:teuthology.orchestra.run.smithi119.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:33:33.387 INFO:teuthology.orchestra.run.smithi186.stderr:bash: line 1: ntpq: command not found 2024-04-22T15:33:33.392 INFO:teuthology.orchestra.run.smithi186.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T15:33:33.392 INFO:teuthology.orchestra.run.smithi186.stdout:=============================================================================== 2024-04-22T15:33:33.392 INFO:teuthology.orchestra.run.smithi186.stdout:^- hv01.front.sepia.ceph.com 2 6 377 60 -1943us[-1943us] +/- 570ms 2024-04-22T15:33:33.393 INFO:teuthology.orchestra.run.smithi186.stdout:^* hv02.front.sepia.ceph.com 3 6 377 62 -569us[ -550us] +/- 54ms 2024-04-22T15:33:33.393 INFO:teuthology.orchestra.run.smithi186.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 62 +691us[ +710us] +/- 65ms 2024-04-22T15:33:33.393 INFO:teuthology.orchestra.run.smithi186.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:33:33.393 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-22T15:33:33.402 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-22T15:33:33.403 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-22T15:33:33.412 DEBUG:teuthology.orchestra.run.smithi119:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-22T15:33:33.495 DEBUG:teuthology.orchestra.run.smithi186:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-22T15:33:33.565 DEBUG:teuthology.orchestra.run.smithi119:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-22T15:33:33.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T15:33:33.592 DEBUG:teuthology.orchestra.run.smithi186:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-22T15:33:33.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T15:33:33.619 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-22T15:33:33.629 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-22T15:33:33.641 INFO:teuthology.task.internal:Duration was 2061.574742 seconds 2024-04-22T15:33:33.641 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-22T15:33:33.651 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-22T15:33:33.651 DEBUG:teuthology.orchestra.run.smithi119:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-22T15:33:33.653 DEBUG:teuthology.orchestra.run.smithi186:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-22T15:33:33.686 INFO:teuthology.orchestra.run.smithi119.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T15:33:33.696 INFO:teuthology.orchestra.run.smithi186.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T15:33:34.058 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-22T15:33:34.058 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi119.front.sepia.ceph.com 2024-04-22T15:33:34.059 DEBUG:teuthology.orchestra.run.smithi119:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-22T15:33:34.078 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi186.front.sepia.ceph.com 2024-04-22T15:33:34.078 DEBUG:teuthology.orchestra.run.smithi186:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-22T15:33:34.126 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-22T15:33:34.126 DEBUG:teuthology.orchestra.run.smithi119:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-22T15:33:34.129 DEBUG:teuthology.orchestra.run.smithi186:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-22T15:33:34.192 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-22T15:33:34.192 DEBUG:teuthology.orchestra.run.smithi119:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-22T15:33:34.194 DEBUG:teuthology.orchestra.run.smithi186:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-22T15:33:48.452 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-22T15:33:48.464 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-22T15:33:48.464 DEBUG:teuthology.orchestra.run.smithi119:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-22T15:33:48.487 DEBUG:teuthology.orchestra.run.smithi186:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-22T15:33:48.516 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-22T15:33:48.527 DEBUG:teuthology.orchestra.run.smithi119:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-22T15:33:48.530 DEBUG:teuthology.orchestra.run.smithi186:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-22T15:33:48.549 INFO:teuthology.orchestra.run.smithi119.stdout:kernel.core_pattern = core 2024-04-22T15:33:48.583 INFO:teuthology.orchestra.run.smithi186.stdout:kernel.core_pattern = core 2024-04-22T15:33:48.673 DEBUG:teuthology.orchestra.run.smithi119:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-22T15:33:48.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T15:33:48.727 DEBUG:teuthology.orchestra.run.smithi186:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-22T15:33:48.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T15:33:48.745 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-22T15:33:48.756 INFO:teuthology.task.internal:Transferring archived files... 2024-04-22T15:33:48.757 DEBUG:teuthology.misc:Transferring archived files from smithi119:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668325/remote/smithi119 2024-04-22T15:33:48.757 DEBUG:teuthology.orchestra.run.smithi119:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-22T15:33:48.830 DEBUG:teuthology.misc:Transferring archived files from smithi186:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668325/remote/smithi186 2024-04-22T15:33:48.831 DEBUG:teuthology.orchestra.run.smithi186:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-22T15:33:48.873 INFO:teuthology.task.internal:Removing archive directory... 2024-04-22T15:33:48.874 DEBUG:teuthology.orchestra.run.smithi119:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-22T15:33:48.876 DEBUG:teuthology.orchestra.run.smithi186:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-22T15:33:48.922 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-22T15:33:48.933 INFO:teuthology.task.internal:Not uploading archives. 2024-04-22T15:33:48.934 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-22T15:33:48.943 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-22T15:33:48.944 DEBUG:teuthology.orchestra.run.smithi119:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-22T15:33:48.946 DEBUG:teuthology.orchestra.run.smithi186:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-22T15:33:48.958 INFO:teuthology.orchestra.run.smithi119.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 22 15:33 /home/ubuntu/cephtest 2024-04-22T15:33:48.978 INFO:teuthology.orchestra.run.smithi186.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 22 15:33 /home/ubuntu/cephtest 2024-04-22T15:33:48.979 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-22T15:33:48.991 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-22T15:33:49.015 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2061.5747423171997 owner: scheduled_adking@teuthology success: true 2024-04-22T15:33:49.015 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-22T15:33:49.113 INFO:teuthology.run:pass