2024-04-22T22:58:24.131 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-22T22:58:24.131 DEBUG:teuthology.run:Teuthology command: teuthology --description orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --archive /home/teuthworker/archive/adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi/7669171 --verbose --name adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi --owner scheduled_adking@teuthology -- /home/teuthworker/archive/adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi/7669171/orig.config.yaml 2024-04-22T22:58:24.168 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-22T22:58:24.249 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi/7669171 branch: wip-adk-testing-2024-04-22-1618 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: adking@redhat.com first_in_suite: false job_id: '7669171' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk-testing-2024-04-22-1618 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 43be020184947e53516056c9931e1ac5bdbbb1a5 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 43be020184947e53516056c9931e1ac5bdbbb1a5 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-adk-testing-2024-04-22-1618 sha1: 43be020184947e53516056c9931e1ac5bdbbb1a5 owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 6551 sha1: 43be020184947e53516056c9931e1ac5bdbbb1a5 sleep_before_teardown: 0 subset: 1/16 suite: orch:cephadm suite_branch: wip-adk-testing-2024-04-22-1618 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_43be020184947e53516056c9931e1ac5bdbbb1a5/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 43be020184947e53516056c9931e1ac5bdbbb1a5 targets: smithi069.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEbgcYyK+npmcxlWH3avzPVUXez6KXbbuijA04aOuqmD47nUmAKmN/EgTTpUkOUrlr0blmrhEuZORB8BgyhnZLg= smithi149.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD6/Ev00AiHj91gkQewicku1LapTrgIgCN79oyVdTIJVghVg/1ibdFyQXoZ5SswRCwxAk7rAbwPse+oQO6LyOzo= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-22_22:45:33 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-04-22T22:58:24.249 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_43be020184947e53516056c9931e1ac5bdbbb1a5/qa; will attempt to use it 2024-04-22T22:58:24.250 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_43be020184947e53516056c9931e1ac5bdbbb1a5/qa/tasks 2024-04-22T22:58:24.250 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-22T22:58:24.251 INFO:teuthology.task.internal:Checking packages... 2024-04-22T22:58:24.274 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '43be020184947e53516056c9931e1ac5bdbbb1a5' 2024-04-22T22:58:24.275 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-22T22:58:24.275 INFO:teuthology.packaging:ref: None 2024-04-22T22:58:24.275 INFO:teuthology.packaging:tag: None 2024-04-22T22:58:24.275 INFO:teuthology.packaging:branch: wip-adk-testing-2024-04-22-1618 2024-04-22T22:58:24.275 INFO:teuthology.packaging:sha1: 43be020184947e53516056c9931e1ac5bdbbb1a5 2024-04-22T22:58:24.275 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk-testing-2024-04-22-1618 2024-04-22T22:58:24.412 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3252.g43be0201 2024-04-22T22:58:24.414 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-22T22:58:24.420 INFO:teuthology.task.internal:no buildpackages task found 2024-04-22T22:58:24.421 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-22T22:58:24.426 INFO:teuthology.task.internal:Saving configuration 2024-04-22T22:58:24.440 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-22T22:58:24.445 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-22T22:58:24.470 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi069.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi/7669171', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 22:52:38.644647', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEbgcYyK+npmcxlWH3avzPVUXez6KXbbuijA04aOuqmD47nUmAKmN/EgTTpUkOUrlr0blmrhEuZORB8BgyhnZLg='} 2024-04-22T22:58:24.491 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi149.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi/7669171', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 22:52:38.646284', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD6/Ev00AiHj91gkQewicku1LapTrgIgCN79oyVdTIJVghVg/1ibdFyQXoZ5SswRCwxAk7rAbwPse+oQO6LyOzo='} 2024-04-22T22:58:24.491 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-22T22:58:24.497 INFO:teuthology.task.internal:roles: ubuntu@smithi069.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-04-22T22:58:24.497 INFO:teuthology.task.internal:roles: ubuntu@smithi149.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-04-22T22:58:24.497 INFO:teuthology.run_tasks:Running task console_log... 2024-04-22T22:58:24.570 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f009de47160>, signals=[15]) 2024-04-22T22:58:24.570 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-22T22:58:24.576 INFO:teuthology.task.internal:Opening connections... 2024-04-22T22:58:24.577 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T22:58:24.578 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T22:58:24.655 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi149.front.sepia.ceph.com 2024-04-22T22:58:24.656 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi149.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T22:58:24.728 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-22T22:58:24.737 DEBUG:teuthology.orchestra.run.smithi069:> uname -m 2024-04-22T22:58:24.755 INFO:teuthology.orchestra.run.smithi069.stdout:x86_64 2024-04-22T22:58:24.755 DEBUG:teuthology.orchestra.run.smithi069:> cat /etc/os-release 2024-04-22T22:58:24.812 INFO:teuthology.orchestra.run.smithi069.stdout:NAME="CentOS Stream" 2024-04-22T22:58:24.812 INFO:teuthology.orchestra.run.smithi069.stdout:VERSION="9" 2024-04-22T22:58:24.812 INFO:teuthology.orchestra.run.smithi069.stdout:ID="centos" 2024-04-22T22:58:24.812 INFO:teuthology.orchestra.run.smithi069.stdout:ID_LIKE="rhel fedora" 2024-04-22T22:58:24.812 INFO:teuthology.orchestra.run.smithi069.stdout:VERSION_ID="9" 2024-04-22T22:58:24.812 INFO:teuthology.orchestra.run.smithi069.stdout:PLATFORM_ID="platform:el9" 2024-04-22T22:58:24.812 INFO:teuthology.orchestra.run.smithi069.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-22T22:58:24.812 INFO:teuthology.orchestra.run.smithi069.stdout:ANSI_COLOR="0;31" 2024-04-22T22:58:24.812 INFO:teuthology.orchestra.run.smithi069.stdout:LOGO="fedora-logo-icon" 2024-04-22T22:58:24.812 INFO:teuthology.orchestra.run.smithi069.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-22T22:58:24.813 INFO:teuthology.orchestra.run.smithi069.stdout:HOME_URL="https://centos.org/" 2024-04-22T22:58:24.813 INFO:teuthology.orchestra.run.smithi069.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-22T22:58:24.813 INFO:teuthology.orchestra.run.smithi069.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-22T22:58:24.813 INFO:teuthology.orchestra.run.smithi069.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-22T22:58:24.813 INFO:teuthology.lock.ops:Updating smithi069.front.sepia.ceph.com on lock server 2024-04-22T22:58:24.841 DEBUG:teuthology.orchestra.run.smithi149:> uname -m 2024-04-22T22:58:24.858 INFO:teuthology.orchestra.run.smithi149.stdout:x86_64 2024-04-22T22:58:24.858 DEBUG:teuthology.orchestra.run.smithi149:> cat /etc/os-release 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:NAME="CentOS Stream" 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:VERSION="9" 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:ID="centos" 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:ID_LIKE="rhel fedora" 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:VERSION_ID="9" 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:PLATFORM_ID="platform:el9" 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:ANSI_COLOR="0;31" 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:LOGO="fedora-logo-icon" 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:HOME_URL="https://centos.org/" 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-22T22:58:24.915 INFO:teuthology.orchestra.run.smithi149.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-22T22:58:24.916 INFO:teuthology.orchestra.run.smithi149.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-22T22:58:24.916 INFO:teuthology.lock.ops:Updating smithi149.front.sepia.ceph.com on lock server 2024-04-22T22:58:24.932 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-22T22:58:24.941 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-22T22:58:24.946 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-22T22:58:24.946 DEBUG:teuthology.orchestra.run.smithi069:> test '!' -e /home/ubuntu/cephtest 2024-04-22T22:58:24.949 DEBUG:teuthology.orchestra.run.smithi149:> test '!' -e /home/ubuntu/cephtest 2024-04-22T22:58:24.970 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-22T22:58:24.975 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-22T22:58:24.975 DEBUG:teuthology.orchestra.run.smithi069:> test -z $(ls -A /var/lib/ceph) 2024-04-22T22:58:25.006 DEBUG:teuthology.orchestra.run.smithi149:> test -z $(ls -A /var/lib/ceph) 2024-04-22T22:58:25.102 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-22T22:58:25.156 INFO:teuthology.run_tasks:Running task kernel... 2024-04-22T22:58:25.170 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-22T22:58:25.170 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-22T22:58:25.171 DEBUG:teuthology.orchestra.run.smithi069:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-22T22:58:25.171 DEBUG:teuthology.orchestra.run.smithi149:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-22T22:58:25.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:58:25.187 DEBUG:teuthology.orchestra.run.smithi149:> uname -r 2024-04-22T22:58:25.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:58:25.188 DEBUG:teuthology.orchestra.run.smithi069:> uname -r 2024-04-22T22:58:25.243 INFO:teuthology.orchestra.run.smithi149.stdout:5.14.0-437.el9.x86_64 2024-04-22T22:58:25.243 INFO:teuthology.task.kernel:Running kernel on smithi149: 5.14.0-437.el9.x86_64 2024-04-22T22:58:25.243 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum install -y kernel 2024-04-22T22:58:25.245 INFO:teuthology.orchestra.run.smithi069.stdout:5.14.0-437.el9.x86_64 2024-04-22T22:58:25.245 INFO:teuthology.task.kernel:Running kernel on smithi069: 5.14.0-437.el9.x86_64 2024-04-22T22:58:25.245 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum install -y kernel 2024-04-22T22:58:27.442 INFO:teuthology.orchestra.run.smithi149.stdout:CentOS Stream 9 - BaseOS 16 MB/s | 8.0 MB 00:00 2024-04-22T22:58:27.457 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.0 MB 00:00 2024-04-22T22:58:30.108 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 9 - AppStream 12 MB/s | 19 MB 00:01 2024-04-22T22:58:31.909 INFO:teuthology.orchestra.run.smithi149.stdout:CentOS Stream 9 - AppStream 5.6 MB/s | 19 MB 00:03 2024-04-22T22:58:35.350 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 9 - CRB 3.9 MB/s | 6.2 MB 00:01 2024-04-22T22:58:36.910 INFO:teuthology.orchestra.run.smithi149.stdout:CentOS Stream 9 - CRB 4.4 MB/s | 6.2 MB 00:01 2024-04-22T22:58:37.134 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 9 - Extras packages 35 kB/s | 16 kB 00:00 2024-04-22T22:58:38.452 INFO:teuthology.orchestra.run.smithi149.stdout:CentOS Stream 9 - Extras packages 74 kB/s | 16 kB 00:00 2024-04-22T22:58:41.837 INFO:teuthology.orchestra.run.smithi149.stdout:Extra Packages for Enterprise Linux 6.4 MB/s | 21 MB 00:03 2024-04-22T22:58:47.022 INFO:teuthology.orchestra.run.smithi149.stdout:lab-extras 14 kB/s | 1.7 kB 00:00 2024-04-22T22:58:48.625 INFO:teuthology.orchestra.run.smithi069.stdout:Extra Packages for Enterprise Linux 1.8 MB/s | 21 MB 00:11 2024-04-22T22:58:48.876 INFO:teuthology.orchestra.run.smithi149.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-22T22:58:48.876 INFO:teuthology.orchestra.run.smithi149.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-22T22:58:48.968 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2024-04-22T22:58:48.969 INFO:teuthology.orchestra.run.smithi149.stdout:Nothing to do. 2024-04-22T22:58:48.969 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2024-04-22T22:58:49.100 DEBUG:teuthology.orchestra.run.smithi149:> echo no | sudo yum reinstall kernel || true 2024-04-22T22:58:49.613 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 10:58:46 PM UTC. 2024-04-22T22:58:49.740 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2024-04-22T22:58:49.754 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2024-04-22T22:58:49.754 INFO:teuthology.orchestra.run.smithi149.stdout: Package Architecture Version Repository Size 2024-04-22T22:58:49.754 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2024-04-22T22:58:49.754 INFO:teuthology.orchestra.run.smithi149.stdout:Reinstalling: 2024-04-22T22:58:49.755 INFO:teuthology.orchestra.run.smithi149.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T22:58:49.755 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T22:58:49.755 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2024-04-22T22:58:49.755 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2024-04-22T22:58:49.755 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T22:58:49.755 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 255 k 2024-04-22T22:58:49.755 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 0 2024-04-22T22:58:49.755 INFO:teuthology.orchestra.run.smithi149.stderr:Operation aborted. 2024-04-22T22:58:49.818 INFO:teuthology.orchestra.run.smithi149.stdout:Is this ok [y/N]: 2024-04-22T22:58:49.819 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum reinstall -y kernel || true 2024-04-22T22:58:50.326 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:00:04 ago on Mon 22 Apr 2024 10:58:46 PM UTC. 2024-04-22T22:58:50.444 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2024-04-22T22:58:50.444 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2024-04-22T22:58:50.445 INFO:teuthology.orchestra.run.smithi149.stdout: Package Architecture Version Repository Size 2024-04-22T22:58:50.445 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2024-04-22T22:58:50.445 INFO:teuthology.orchestra.run.smithi149.stdout:Reinstalling: 2024-04-22T22:58:50.445 INFO:teuthology.orchestra.run.smithi149.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T22:58:50.445 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T22:58:50.445 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2024-04-22T22:58:50.445 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2024-04-22T22:58:50.445 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T22:58:50.445 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 255 k 2024-04-22T22:58:50.445 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 0 2024-04-22T22:58:50.445 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2024-04-22T22:58:50.963 INFO:teuthology.orchestra.run.smithi149.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.0 MB/s | 255 kB 00:00 2024-04-22T22:58:50.964 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2024-04-22T22:58:50.964 INFO:teuthology.orchestra.run.smithi149.stdout:Total 491 kB/s | 255 kB 00:00 2024-04-22T22:58:50.986 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2024-04-22T22:58:51.186 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2024-04-22T22:58:51.186 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2024-04-22T22:58:51.191 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2024-04-22T22:58:51.191 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2024-04-22T22:58:51.329 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2024-04-22T22:58:51.406 INFO:teuthology.orchestra.run.smithi149.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T22:58:51.406 INFO:teuthology.orchestra.run.smithi149.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T22:58:52.100 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T22:58:52.101 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T22:58:52.814 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T22:58:52.815 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T22:58:52.815 INFO:teuthology.orchestra.run.smithi149.stdout:Reinstalled: 2024-04-22T22:58:52.815 INFO:teuthology.orchestra.run.smithi149.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-22T22:58:52.815 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T22:58:52.815 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2024-04-22T22:58:52.979 DEBUG:teuthology.orchestra.run.smithi149:> rpm -q kernel | sort -rV | head -n 1 2024-04-22T22:58:53.026 INFO:teuthology.orchestra.run.smithi149.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-22T22:58:53.027 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-22T22:58:53.027 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-22T22:58:53.027 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-04-22T22:58:53.027 DEBUG:teuthology.orchestra.run.smithi149:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-22T22:58:53.067 INFO:teuthology.orchestra.run.smithi149.stdout:ttyS1 2024-04-22T22:58:53.089 DEBUG:teuthology.parallel:result is None 2024-04-22T22:58:53.701 INFO:teuthology.orchestra.run.smithi069.stdout:lab-extras 10 kB/s | 1.7 kB 00:00 2024-04-22T22:58:55.565 INFO:teuthology.orchestra.run.smithi069.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-22T22:58:55.565 INFO:teuthology.orchestra.run.smithi069.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-22T22:58:55.650 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2024-04-22T22:58:55.651 INFO:teuthology.orchestra.run.smithi069.stdout:Nothing to do. 2024-04-22T22:58:55.651 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2024-04-22T22:58:55.749 DEBUG:teuthology.orchestra.run.smithi069:> echo no | sudo yum reinstall kernel || true 2024-04-22T22:58:56.246 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 10:58:53 PM UTC. 2024-04-22T22:58:56.365 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2024-04-22T22:58:56.379 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-22T22:58:56.379 INFO:teuthology.orchestra.run.smithi069.stdout: Package Architecture Version Repository Size 2024-04-22T22:58:56.379 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-22T22:58:56.379 INFO:teuthology.orchestra.run.smithi069.stdout:Reinstalling: 2024-04-22T22:58:56.379 INFO:teuthology.orchestra.run.smithi069.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T22:58:56.379 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T22:58:56.380 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2024-04-22T22:58:56.380 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-22T22:58:56.380 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T22:58:56.380 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 255 k 2024-04-22T22:58:56.380 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 0 2024-04-22T22:58:56.380 INFO:teuthology.orchestra.run.smithi069.stderr:Operation aborted. 2024-04-22T22:58:56.433 INFO:teuthology.orchestra.run.smithi069.stdout:Is this ok [y/N]: 2024-04-22T22:58:56.434 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum reinstall -y kernel || true 2024-04-22T22:58:56.924 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 10:58:53 PM UTC. 2024-04-22T22:58:57.041 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2024-04-22T22:58:57.041 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-22T22:58:57.041 INFO:teuthology.orchestra.run.smithi069.stdout: Package Architecture Version Repository Size 2024-04-22T22:58:57.041 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-22T22:58:57.041 INFO:teuthology.orchestra.run.smithi069.stdout:Reinstalling: 2024-04-22T22:58:57.042 INFO:teuthology.orchestra.run.smithi069.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T22:58:57.042 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T22:58:57.042 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2024-04-22T22:58:57.042 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-22T22:58:57.042 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T22:58:57.042 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 255 k 2024-04-22T22:58:57.043 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 0 2024-04-22T22:58:57.043 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2024-04-22T22:58:57.210 INFO:teuthology.orchestra.run.smithi069.stdout:kernel-5.14.0-437.el9.x86_64.rpm 2.4 MB/s | 255 kB 00:00 2024-04-22T22:58:57.211 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2024-04-22T22:58:57.211 INFO:teuthology.orchestra.run.smithi069.stdout:Total 1.5 MB/s | 255 kB 00:00 2024-04-22T22:58:57.229 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2024-04-22T22:58:57.412 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2024-04-22T22:58:57.412 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2024-04-22T22:58:57.417 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2024-04-22T22:58:57.418 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2024-04-22T22:58:57.564 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2024-04-22T22:58:57.653 INFO:teuthology.orchestra.run.smithi069.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T22:58:57.653 INFO:teuthology.orchestra.run.smithi069.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T22:58:58.325 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T22:58:58.325 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T22:58:58.984 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T22:58:58.984 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T22:58:58.984 INFO:teuthology.orchestra.run.smithi069.stdout:Reinstalled: 2024-04-22T22:58:58.984 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-22T22:58:58.984 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T22:58:58.984 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2024-04-22T22:58:59.138 DEBUG:teuthology.orchestra.run.smithi069:> rpm -q kernel | sort -rV | head -n 1 2024-04-22T22:58:59.184 INFO:teuthology.orchestra.run.smithi069.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-22T22:58:59.184 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-22T22:58:59.184 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-22T22:58:59.184 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-04-22T22:58:59.184 DEBUG:teuthology.orchestra.run.smithi069:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-22T22:58:59.217 INFO:teuthology.orchestra.run.smithi069.stdout:ttyS1 2024-04-22T22:58:59.243 DEBUG:teuthology.parallel:result is None 2024-04-22T22:58:59.244 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-22T22:58:59.252 INFO:teuthology.task.internal:Creating test directory... 2024-04-22T22:58:59.252 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-22T22:58:59.286 DEBUG:teuthology.orchestra.run.smithi149:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-22T22:58:59.304 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-22T22:58:59.310 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-22T22:58:59.321 INFO:teuthology.task.internal:Creating archive directory... 2024-04-22T22:58:59.321 DEBUG:teuthology.orchestra.run.smithi069:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-22T22:58:59.342 DEBUG:teuthology.orchestra.run.smithi149:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-22T22:58:59.372 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-22T22:58:59.379 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-22T22:58:59.379 DEBUG:teuthology.orchestra.run.smithi069:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-22T22:58:59.404 DEBUG:teuthology.orchestra.run.smithi149:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-22T22:58:59.431 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T22:58:59.440 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T22:58:59.440 INFO:teuthology.orchestra.run.smithi149.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T22:58:59.451 INFO:teuthology.orchestra.run.smithi149.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T22:58:59.453 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-22T22:58:59.460 INFO:teuthology.task.internal:Configuring sudo... 2024-04-22T22:58:59.460 DEBUG:teuthology.orchestra.run.smithi069:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-22T22:58:59.484 DEBUG:teuthology.orchestra.run.smithi149:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-22T22:58:59.521 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-22T22:58:59.529 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-22T22:58:59.529 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-22T22:58:59.548 DEBUG:teuthology.orchestra.run.smithi149:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-22T22:58:59.578 DEBUG:teuthology.orchestra.run.smithi069:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T22:58:59.640 DEBUG:teuthology.orchestra.run.smithi069:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T22:58:59.686 DEBUG:teuthology.orchestra.run.smithi069:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T22:58:59.768 DEBUG:teuthology.orchestra.run.smithi069:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T22:58:59.806 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-22T22:58:59.806 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-22T22:58:59.868 DEBUG:teuthology.orchestra.run.smithi149:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T22:58:59.914 DEBUG:teuthology.orchestra.run.smithi149:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T22:58:59.959 DEBUG:teuthology.orchestra.run.smithi149:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T22:59:00.043 DEBUG:teuthology.orchestra.run.smithi149:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T22:59:00.085 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2024-04-22T22:59:00.085 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-22T22:59:00.151 DEBUG:teuthology.orchestra.run.smithi069:> sudo service rsyslog restart 2024-04-22T22:59:00.154 DEBUG:teuthology.orchestra.run.smithi149:> sudo service rsyslog restart 2024-04-22T22:59:00.186 INFO:teuthology.orchestra.run.smithi069.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T22:59:00.239 INFO:teuthology.orchestra.run.smithi149.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T22:59:00.740 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-22T22:59:00.749 INFO:teuthology.task.internal:Starting timer... 2024-04-22T22:59:00.749 INFO:teuthology.run_tasks:Running task pcp... 2024-04-22T22:59:00.763 INFO:teuthology.run_tasks:Running task selinux... 2024-04-22T22:59:00.772 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-04-22T22:59:00.772 DEBUG:teuthology.orchestra.run.smithi069:> sudo service auditd rotate 2024-04-22T22:59:00.815 INFO:teuthology.orchestra.run.smithi069.stdout:Rotating logs: 2024-04-22T22:59:00.816 DEBUG:teuthology.orchestra.run.smithi149:> sudo service auditd rotate 2024-04-22T22:59:00.888 INFO:teuthology.orchestra.run.smithi149.stdout:Rotating logs: 2024-04-22T22:59:00.891 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-22T22:59:00.891 DEBUG:teuthology.orchestra.run.smithi069:> /usr/sbin/getenforce 2024-04-22T22:59:00.922 INFO:teuthology.orchestra.run.smithi069.stdout:Permissive 2024-04-22T22:59:00.923 DEBUG:teuthology.orchestra.run.smithi149:> /usr/sbin/getenforce 2024-04-22T22:59:00.958 INFO:teuthology.orchestra.run.smithi149.stdout:Permissive 2024-04-22T22:59:00.958 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi069.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi149.front.sepia.ceph.com': 'permissive'} 2024-04-22T22:59:00.959 DEBUG:teuthology.orchestra.run.smithi069:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-22T22:59:00.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:59:00.988 DEBUG:teuthology.orchestra.run.smithi149:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-22T22:59:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:59:01.024 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-22T22:59:01.024 DEBUG:teuthology.orchestra.run.smithi069:> sudo /usr/sbin/setenforce permissive 2024-04-22T22:59:01.052 DEBUG:teuthology.orchestra.run.smithi149:> sudo /usr/sbin/setenforce permissive 2024-04-22T22:59:01.088 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-22T22:59:01.099 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-22T22:59:01.114 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-22T22:59:01.116 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi069.front.sepia.ceph.com,smithi149.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-22T23:02:36.151 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi069.front.sepia.ceph.com'), Remote(name='ubuntu@smithi149.front.sepia.ceph.com')] 2024-04-22T23:02:36.152 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-04-22T23:02:36.153 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T23:02:36.270 DEBUG:teuthology.orchestra.run.smithi069:> true 2024-04-22T23:02:36.353 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-04-22T23:02:36.354 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi149.front.sepia.ceph.com' 2024-04-22T23:02:36.354 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi149.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T23:02:36.429 DEBUG:teuthology.orchestra.run.smithi149:> true 2024-04-22T23:02:36.466 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi149.front.sepia.ceph.com' 2024-04-22T23:02:36.466 INFO:teuthology.run_tasks:Running task clock... 2024-04-22T23:02:36.477 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-22T23:02:36.477 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-22T23:02:36.478 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T23:02:36.480 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-22T23:02:36.481 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T23:02:36.505 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-22T23:02:36.518 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-22T23:02:36.535 INFO:teuthology.orchestra.run.smithi149.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-22T23:02:36.541 INFO:teuthology.orchestra.run.smithi069.stderr:sudo: ntpd: command not found 2024-04-22T23:02:36.548 INFO:teuthology.orchestra.run.smithi149.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-22T23:02:36.551 INFO:teuthology.orchestra.run.smithi069.stdout:506 Cannot talk to daemon 2024-04-22T23:02:36.564 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-22T23:02:36.573 INFO:teuthology.orchestra.run.smithi149.stderr:sudo: ntpd: command not found 2024-04-22T23:02:36.577 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-22T23:02:36.581 INFO:teuthology.orchestra.run.smithi149.stdout:506 Cannot talk to daemon 2024-04-22T23:02:36.594 INFO:teuthology.orchestra.run.smithi149.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-22T23:02:36.607 INFO:teuthology.orchestra.run.smithi149.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-22T23:02:36.631 INFO:teuthology.orchestra.run.smithi069.stderr:bash: line 1: ntpq: command not found 2024-04-22T23:02:36.636 INFO:teuthology.orchestra.run.smithi069.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T23:02:36.636 INFO:teuthology.orchestra.run.smithi069.stdout:=============================================================================== 2024-04-22T23:02:36.636 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T23:02:36.636 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T23:02:36.636 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T23:02:36.636 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T23:02:36.660 INFO:teuthology.orchestra.run.smithi149.stderr:bash: line 1: ntpq: command not found 2024-04-22T23:02:36.664 INFO:teuthology.orchestra.run.smithi149.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T23:02:36.664 INFO:teuthology.orchestra.run.smithi149.stdout:=============================================================================== 2024-04-22T23:02:36.664 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T23:02:36.664 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T23:02:36.664 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T23:02:36.664 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T23:02:36.665 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-22T23:02:36.781 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '43be020184947e53516056c9931e1ac5bdbbb1a5'} 2024-04-22T23:02:36.781 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-22T23:02:36.782 INFO:tasks.cephadm:Cluster fsid is 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:02:36.782 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-22T23:02:36.782 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-04-22T23:02:36.782 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi069': '172.21.15.69', 'mon.smithi149': '172.21.15.149'} 2024-04-22T23:02:36.782 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-22T23:02:36.783 DEBUG:teuthology.orchestra.run.smithi069:> sudo hostname $(hostname -s) 2024-04-22T23:02:36.807 DEBUG:teuthology.orchestra.run.smithi149:> sudo hostname $(hostname -s) 2024-04-22T23:02:36.832 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-22T23:02:36.833 DEBUG:teuthology.orchestra.run.smithi069:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-22T23:02:37.075 INFO:teuthology.orchestra.run.smithi069.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 22 23:02 /home/ubuntu/cephtest/cephadm 2024-04-22T23:02:37.075 DEBUG:teuthology.orchestra.run.smithi149:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-22T23:02:37.315 INFO:teuthology.orchestra.run.smithi149.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 22 23:02 /home/ubuntu/cephtest/cephadm 2024-04-22T23:02:37.315 DEBUG:teuthology.orchestra.run.smithi069:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-22T23:02:37.331 DEBUG:teuthology.orchestra.run.smithi149:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-22T23:02:37.355 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-22T23:02:37.355 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-22T23:02:37.373 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-22T23:02:38.856 INFO:teuthology.orchestra.run.smithi149.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T23:02:38.930 INFO:teuthology.orchestra.run.smithi069.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T23:03:00.026 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:03:00.026 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-22T23:03:00.026 INFO:teuthology.orchestra.run.smithi069.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-22T23:03:00.026 INFO:teuthology.orchestra.run.smithi069.stdout: "repo_digests": [ 2024-04-22T23:03:00.026 INFO:teuthology.orchestra.run.smithi069.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-22T23:03:00.026 INFO:teuthology.orchestra.run.smithi069.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-22T23:03:00.026 INFO:teuthology.orchestra.run.smithi069.stdout: ] 2024-04-22T23:03:00.027 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:03:03.295 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2024-04-22T23:03:03.295 INFO:teuthology.orchestra.run.smithi149.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-22T23:03:03.296 INFO:teuthology.orchestra.run.smithi149.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-22T23:03:03.296 INFO:teuthology.orchestra.run.smithi149.stdout: "repo_digests": [ 2024-04-22T23:03:03.296 INFO:teuthology.orchestra.run.smithi149.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-22T23:03:03.296 INFO:teuthology.orchestra.run.smithi149.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-22T23:03:03.296 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2024-04-22T23:03:03.296 INFO:teuthology.orchestra.run.smithi149.stdout:} 2024-04-22T23:03:03.313 DEBUG:teuthology.orchestra.run.smithi069:> sudo mkdir -p /etc/ceph 2024-04-22T23:03:03.345 DEBUG:teuthology.orchestra.run.smithi149:> sudo mkdir -p /etc/ceph 2024-04-22T23:03:03.370 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod 777 /etc/ceph 2024-04-22T23:03:03.412 DEBUG:teuthology.orchestra.run.smithi149:> sudo chmod 777 /etc/ceph 2024-04-22T23:03:03.435 INFO:tasks.cephadm:Writing seed config... 2024-04-22T23:03:03.437 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-22T23:03:03.437 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-22T23:03:03.437 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-22T23:03:03.437 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-22T23:03:03.437 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-22T23:03:03.437 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-22T23:03:03.437 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-22T23:03:03.437 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-22T23:03:03.438 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-22T23:03:03.438 DEBUG:teuthology.orchestra.run.smithi069:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-22T23:03:03.524 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 6928ed7e-00fc-11ef-bc93-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-22T23:03:03.525 DEBUG:teuthology.orchestra.run.smithi069:mon.smithi069> sudo journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi069.service 2024-04-22T23:03:03.568 INFO:tasks.cephadm:Bootstrapping... 2024-04-22T23:03:03.568 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.69 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-22T23:03:03.766 INFO:teuthology.orchestra.run.smithi069.stderr:-------------------------------------------------------------------------------- 2024-04-22T23:03:03.766 INFO:teuthology.orchestra.run.smithi069.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '6928ed7e-00fc-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.69', '--skip-admin-label'] 2024-04-22T23:03:04.226 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: 4.9.4-dev 2024-04-22T23:03:04.228 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying podman|docker is present... 2024-04-22T23:03:04.250 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: 4.9.4-dev 2024-04-22T23:03:04.253 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying lvm2 is present... 2024-04-22T23:03:04.253 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying time synchronization is in place... 2024-04-22T23:03:04.260 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-22T23:03:04.266 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-04-22T23:03:04.273 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: enabled 2024-04-22T23:03:04.279 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: active 2024-04-22T23:03:04.280 INFO:teuthology.orchestra.run.smithi069.stderr:Unit chronyd.service is enabled and running 2024-04-22T23:03:04.280 INFO:teuthology.orchestra.run.smithi069.stderr:Repeating the final host check... 2024-04-22T23:03:04.301 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: 4.9.4-dev 2024-04-22T23:03:04.304 INFO:teuthology.orchestra.run.smithi069.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-22T23:03:04.304 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl is present 2024-04-22T23:03:04.304 INFO:teuthology.orchestra.run.smithi069.stderr:lvcreate is present 2024-04-22T23:03:04.310 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-22T23:03:04.316 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-04-22T23:03:04.323 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: enabled 2024-04-22T23:03:04.329 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: active 2024-04-22T23:03:04.330 INFO:teuthology.orchestra.run.smithi069.stderr:Unit chronyd.service is enabled and running 2024-04-22T23:03:04.330 INFO:teuthology.orchestra.run.smithi069.stderr:Host looks OK 2024-04-22T23:03:04.330 INFO:teuthology.orchestra.run.smithi069.stderr:Cluster fsid: 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:03:04.330 INFO:teuthology.orchestra.run.smithi069.stderr:Acquiring lock 140280312255776 on /run/cephadm/6928ed7e-00fc-11ef-bc93-c7b262605968.lock 2024-04-22T23:03:04.331 INFO:teuthology.orchestra.run.smithi069.stderr:Lock 140280312255776 acquired on /run/cephadm/6928ed7e-00fc-11ef-bc93-c7b262605968.lock 2024-04-22T23:03:04.331 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying IP 172.21.15.69 port 3300 ... 2024-04-22T23:03:04.331 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying IP 172.21.15.69 port 6789 ... 2024-04-22T23:03:04.331 INFO:teuthology.orchestra.run.smithi069.stderr:Base mon IP is 172.21.15.69, final addrv is [v2:172.21.15.69:3300,v1:172.21.15.69:6789] 2024-04-22T23:03:04.333 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.69 metric 100 2024-04-22T23:03:04.334 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.69 metric 100 2024-04-22T23:03:04.336 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-22T23:03:04.336 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-04-22T23:03:04.336 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-04-22T23:03:04.339 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-22T23:03:04.340 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-22T23:03:04.340 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-22T23:03:04.340 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-22T23:03:04.340 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: inet6 fe80::744a:9e75:3d79:38d4/64 scope link noprefixroute 2024-04-22T23:03:04.340 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-22T23:03:04.340 INFO:teuthology.orchestra.run.smithi069.stderr:Mon IP `172.21.15.69` is in CIDR network `172.21.0.0/20` 2024-04-22T23:03:04.340 INFO:teuthology.orchestra.run.smithi069.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-22T23:03:04.341 INFO:teuthology.orchestra.run.smithi069.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T23:03:04.447 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-22T23:03:04.792 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Getting image source signatures 2024-04-22T23:03:04.792 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-22T23:03:04.792 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-22T23:03:04.792 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-22T23:03:04.792 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-22T23:03:04.792 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-22T23:03:04.792 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-22T23:03:05.752 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Writing manifest to image destination 2024-04-22T23:03:05.914 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-22T23:03:06.274 INFO:teuthology.orchestra.run.smithi069.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-22T23:03:06.720 INFO:teuthology.orchestra.run.smithi069.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-22T23:03:06.720 INFO:teuthology.orchestra.run.smithi069.stderr:Extracting ceph user uid/gid from container image... 2024-04-22T23:03:07.094 INFO:teuthology.orchestra.run.smithi069.stderr:stat: 167 167 2024-04-22T23:03:07.683 INFO:teuthology.orchestra.run.smithi069.stderr:Creating initial keys... 2024-04-22T23:03:08.083 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph-authtool: AQCr7CZm/oJoOxAALUtyASPrD3m4IveVtyGbbA== 2024-04-22T23:03:08.963 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph-authtool: AQCs7CZm2Ci2NBAAYRO3tDO9bolN5KSC50Gm8g== 2024-04-22T23:03:09.860 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph-authtool: AQCt7CZmdTGlLRAAz6vxKsnRa0oObOCJN+8Ztw== 2024-04-22T23:03:13.982 INFO:teuthology.orchestra.run.smithi069.stderr:Creating initial monmap... 2024-04-22T23:03:14.925 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-22T23:03:14.925 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-22T23:03:14.925 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:03:14.925 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-22T23:03:15.364 INFO:teuthology.orchestra.run.smithi069.stderr:monmaptool for smithi069 [v2:172.21.15.69:3300,v1:172.21.15.69:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-22T23:03:15.365 INFO:teuthology.orchestra.run.smithi069.stderr:setting min_mon_release = octopus 2024-04-22T23:03:15.365 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: set fsid to 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:03:15.365 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-22T23:03:15.365 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-22T23:03:15.365 INFO:teuthology.orchestra.run.smithi069.stderr:Creating mon... 2024-04-22T23:03:16.495 INFO:teuthology.orchestra.run.smithi069.stderr:create mon.smithi069 on 2024-04-22T23:03:16.701 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-22T23:03:16.837 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-6928ed7e-00fc-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-6928ed7e-00fc-11ef-bc93-c7b262605968.target. 2024-04-22T23:03:16.837 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-6928ed7e-00fc-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-6928ed7e-00fc-11ef-bc93-c7b262605968.target. 2024-04-22T23:03:17.134 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Failed to reset failed state of unit ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi069.service: Unit ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi069.service not loaded. 2024-04-22T23:03:17.143 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/ceph-6928ed7e-00fc-11ef-bc93-c7b262605968.target.wants/ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi069.service → /etc/systemd/system/ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@.service. 2024-04-22T23:03:17.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:17 smithi069 podman[21486]: 2024-04-22 23:03:17.476367614 +0000 UTC m=+0.018771839 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-22T23:03:17.835 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: disabled 2024-04-22T23:03:17.841 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-04-22T23:03:17.842 INFO:teuthology.orchestra.run.smithi069.stderr:firewalld.service is not enabled 2024-04-22T23:03:17.842 INFO:teuthology.orchestra.run.smithi069.stderr:Not possible to enable service . firewalld.service is not available 2024-04-22T23:03:17.842 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mon to start... 2024-04-22T23:03:17.843 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mon... 2024-04-22T23:03:17.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:17 smithi069 podman[21486]: 2024-04-22 23:03:17.745898285 +0000 UTC m=+0.288302493 container init b7f14ec8b24daed2f610e23dc8013ad37148026951f85701acf15fcd9aa14b11 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, version=8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, vendor=Red Hat, Inc., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754) 2024-04-22T23:03:17.969 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:17 smithi069 podman[21486]: 2024-04-22 23:03:17.748833465 +0000 UTC m=+0.291237664 container start b7f14ec8b24daed2f610e23dc8013ad37148026951f85701acf15fcd9aa14b11 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_CLEAN=True, vcs-type=git, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, architecture=x86_64, maintainer=Guillaume Abrioux ) 2024-04-22T23:03:17.969 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:17 smithi069 ceph-mon[21512]: mkfs 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:03:17.969 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:17 smithi069 ceph-mon[21512]: mon.smithi069 is new leader, mons smithi069 in quorum (ranks 0) 2024-04-22T23:03:17.969 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:17 smithi069 bash[21486]: b7f14ec8b24daed2f610e23dc8013ad37148026951f85701acf15fcd9aa14b11 2024-04-22T23:03:17.969 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:17 smithi069 systemd[1]: Started Ceph mon.smithi069 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:03:18.379 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: cluster: 2024-04-22T23:03:18.379 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: id: 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:03:18.379 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-22T23:03:18.379 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:18.379 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: services: 2024-04-22T23:03:18.379 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi069 (age 0.59094s) 2024-04-22T23:03:18.379 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-22T23:03:18.380 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-22T23:03:18.380 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:18.380 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: data: 2024-04-22T23:03:18.380 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-22T23:03:18.380 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-22T23:03:18.380 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-22T23:03:18.380 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: pgs: 2024-04-22T23:03:18.380 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:18.934 INFO:teuthology.orchestra.run.smithi069.stderr:mon is available 2024-04-22T23:03:18.934 INFO:teuthology.orchestra.run.smithi069.stderr:Assimilating anything we can from ceph.conf... 2024-04-22T23:03:19.046 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:18 smithi069 ceph-mon[21512]: mon.smithi069 is new leader, mons smithi069 in quorum (ranks 0) 2024-04-22T23:03:19.046 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:18 smithi069 ceph-mon[21512]: monmap e1: 1 mons at {smithi069=[v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0]} 2024-04-22T23:03:19.046 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:18 smithi069 ceph-mon[21512]: fsmap 2024-04-22T23:03:19.046 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:18 smithi069 ceph-mon[21512]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T23:03:19.046 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:18 smithi069 ceph-mon[21512]: mgrmap e1: no daemons active 2024-04-22T23:03:19.046 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:18 smithi069 ceph-mon[21512]: from='client.? 172.21.15.69:0/2417584064' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T23:03:19.930 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:19.930 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [global] 2024-04-22T23:03:19.930 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: fsid = 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:03:19.930 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.69:3300,v1:172.21.15.69:6789] 2024-04-22T23:03:19.930 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-22T23:03:19.930 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-22T23:03:19.931 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-22T23:03:19.931 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-22T23:03:19.931 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:19.931 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [mgr] 2024-04-22T23:03:19.931 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-22T23:03:19.931 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:19.931 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [osd] 2024-04-22T23:03:19.931 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-22T23:03:19.931 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-22T23:03:20.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:19 smithi069 ceph-mon[21512]: from='client.? 172.21.15.69:0/1404625787' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T23:03:20.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:19 smithi069 ceph-mon[21512]: from='client.? 172.21.15.69:0/1404625787' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-22T23:03:20.763 INFO:teuthology.orchestra.run.smithi069.stderr:Generating new minimal ceph.conf... 2024-04-22T23:03:21.607 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:21 smithi069 ceph-mon[21512]: from='client.? 172.21.15.69:0/1502316220' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:03:21.894 INFO:teuthology.orchestra.run.smithi069.stderr:Restarting the monitor... 2024-04-22T23:03:22.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:21 smithi069 systemd[1]: Stopping Ceph mon.smithi069 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:03:22.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:21 smithi069 bash[21732]: Error: no container with name or ID "ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon.smithi069" found: no such container 2024-04-22T23:03:22.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:22 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069[21506]: 2024-04-22T23:03:22.059+0000 7f3f5717b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi069 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:03:22.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:22 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069[21506]: 2024-04-22T23:03:22.059+0000 7f3f5717b700 -1 mon.smithi069@0(leader) e1 *** Got Signal Terminated *** 2024-04-22T23:03:22.552 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:22 smithi069 podman[21745]: 2024-04-22 23:03:22.264294477 +0000 UTC m=+0.316076364 container died b7f14ec8b24daed2f610e23dc8013ad37148026951f85701acf15fcd9aa14b11 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, name=centos-stream, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, architecture=x86_64, distribution-scope=public, version=8) 2024-04-22T23:03:22.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:22 smithi069 podman[21745]: 2024-04-22 23:03:22.650949875 +0000 UTC m=+0.702731756 container cleanup b7f14ec8b24daed2f610e23dc8013ad37148026951f85701acf15fcd9aa14b11 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, ceph=True, version=8, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.buildah.version=1.19.8, GIT_CLEAN=True) 2024-04-22T23:03:22.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:22 smithi069 bash[21745]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069 2024-04-22T23:03:22.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:22 smithi069 bash[21773]: Error: no container with name or ID "ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon.smithi069" found: no such container 2024-04-22T23:03:23.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:22 smithi069 podman[21759]: 2024-04-22 23:03:22.926756494 +0000 UTC m=+0.659624894 container remove b7f14ec8b24daed2f610e23dc8013ad37148026951f85701acf15fcd9aa14b11 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., ceph=True, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, name=centos-stream, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8) 2024-04-22T23:03:23.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:22 smithi069 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi069.service: Deactivated successfully. 2024-04-22T23:03:23.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:22 smithi069 systemd[1]: Stopped Ceph mon.smithi069 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:03:23.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:22 smithi069 systemd[1]: Starting Ceph mon.smithi069 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:03:23.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 podman[21840]: 2024-04-22 23:03:23.055955312 +0000 UTC m=+0.017966973 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-22T23:03:23.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 podman[21840]: 2024-04-22 23:03:23.169211766 +0000 UTC m=+0.131223416 container create 008ad8fe38c6996b05863941b6022e3e37280cf81fb93b1d02ce3e90d6e4800d (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, ceph=True, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, distribution-scope=public, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T23:03:23.434 INFO:teuthology.orchestra.run.smithi069.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-22T23:03:23.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 podman[21840]: 2024-04-22 23:03:23.349203726 +0000 UTC m=+0.311215385 container init 008ad8fe38c6996b05863941b6022e3e37280cf81fb93b1d02ce3e90d6e4800d (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, vcs-type=git, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, ceph=True, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.buildah.version=1.19.8, name=centos-stream, maintainer=Guillaume Abrioux ) 2024-04-22T23:03:23.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 podman[21840]: 2024-04-22 23:03:23.352375748 +0000 UTC m=+0.314387405 container start 008ad8fe38c6996b05863941b6022e3e37280cf81fb93b1d02ce3e90d6e4800d (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, maintainer=Guillaume Abrioux , version=8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vendor=Red Hat, Inc.) 2024-04-22T23:03:23.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T23:03:23.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: pidfile_write: ignore empty --pid-file 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: load: jerasure load: lrc 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: RocksDB version: 6.15.5 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Compile date Apr 18 2022 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: DB SUMMARY 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: DB Session ID: W3QU6T0F9FJ24UQL7OBS 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: CURRENT file: CURRENT 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi069/store.db dir, Total Num: 1, files: 000008.sst 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi069/store.db: 000010.log size: 82713 ; 2024-04-22T23:03:23.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.error_if_exists: 0 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.create_if_missing: 0 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.paranoid_checks: 1 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.env: 0x55af40ec6860 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.fs: Posix File System 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.info_log: 0x55af433bbfa0 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.statistics: (nil) 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.use_fsync: 0 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_log_file_size: 0 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T23:03:23.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.allow_fallocate: 1 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.use_direct_reads: 0 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.db_log_dir: 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi069/store.db 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T23:03:23.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T23:03:23.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T23:03:23.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T23:03:23.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T23:03:23.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.write_buffer_manager: 0x55af434ac330 2024-04-22T23:03:23.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T23:03:23.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-22T23:03:23.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T23:03:23.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T23:03:23.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.unordered_write: 0 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.row_cache: None 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.wal_filter: None 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T23:03:23.652 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T23:03:23.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.preserve_deletes: 0 2024-04-22T23:03:23.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.two_write_queues: 0 2024-04-22T23:03:23.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T23:03:23.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.atomic_flush: 0 2024-04-22T23:03:23.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T23:03:23.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T23:03:23.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T23:03:23.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.log_readahead_size: 0 2024-04-22T23:03:23.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T23:03:23.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T23:03:23.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T23:03:23.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_background_jobs: 2 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_background_compactions: -1 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_subcompactions: 1 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T23:03:23.654 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T23:03:23.655 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_open_files: -1 2024-04-22T23:03:23.655 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T23:03:23.655 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T23:03:23.655 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T23:03:23.655 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T23:03:23.655 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_background_flushes: -1 2024-04-22T23:03:23.655 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Compression algorithms supported: 2024-04-22T23:03:23.655 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T23:03:23.655 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: kZSTD supported: 0 2024-04-22T23:03:23.655 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: kXpressCompression supported: 0 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: kLZ4Compression supported: 1 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: kBZip2Compression supported: 0 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: kZlibCompression supported: 1 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: kSnappyCompression supported: 1 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi069/store.db/MANIFEST-000009 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.merge_operator: 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_filter: None 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T23:03:23.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55af43387d00) 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: cache_index_and_filter_blocks: 1 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: pin_top_level_index_and_filter: 1 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_type: 0 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: data_block_index_type: 0 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_shortening: 1 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: hash_index_allow_collision: 1 2024-04-22T23:03:23.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: checksum: 1 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: no_block_cache: 0 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache: 0x55af433f2170 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_name: BinnedLRUCache 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_options: 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: capacity : 536870912 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: num_shard_bits : 4 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: strict_capacity_limit : 0 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: high_pri_pool_ratio: 0.000 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_compressed: (nil) 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: persistent_cache: (nil) 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_size: 4096 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_size_deviation: 10 2024-04-22T23:03:23.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_restart_interval: 16 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_block_restart_interval: 1 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: metadata_block_size: 4096 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: partition_filters: 0 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: use_delta_encoding: 1 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: whole_key_filtering: 1 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: verify_compression: 0 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: read_amp_bytes_per_bit: 0 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: format_version: 4 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: enable_index_compression: 1 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_align: 0 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T23:03:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compression: NoCompression 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.num_levels: 7 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T23:03:23.660 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T23:03:23.661 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T23:03:23.661 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T23:03:23.661 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T23:03:23.661 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T23:03:23.661 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T23:03:23.661 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T23:03:23.661 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T23:03:23.661 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T23:03:23.661 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T23:03:23.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.arena_block_size: 4194304 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T23:03:23.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.table_properties_collectors: 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.inplace_update_support: 0 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.bloom_locality: 0 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.max_successive_merges: 0 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T23:03:23.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.ttl: 2592000 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.enable_blob_files: false 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.min_blob_size: 0 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi069/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713827003377174, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-22T23:03:23.665 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-22T23:03:23.666 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-22T23:03:23.666 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713827003377877, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79395, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77659, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71968, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713827003, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "4f7ca0af-9b79-4405-8e30-f828ef24bef8", "db_session_id": "W3QU6T0F9FJ24UQL7OBS"}} 2024-04-22T23:03:23.666 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-22T23:03:23.666 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713827003378240, "job": 1, "event": "recovery_finished"} 2024-04-22T23:03:23.666 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi069/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T23:03:23.666 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55af433d8a80 2024-04-22T23:03:23.666 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: DB pointer 0x55af433e8000 2024-04-22T23:03:23.666 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-22T23:03:23.666 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-22T23:03:23.666 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** DB Stats ** 2024-04-22T23:03:23.667 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T23:03:23.667 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T23:03:23.667 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T23:03:23.667 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T23:03:23.667 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T23:03:23.667 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-22T23:03:23.667 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T23:03:23.667 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:03:23.667 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-04-22T23:03:23.668 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T23:03:23.668 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T23:03:23.668 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.3 0.00 0.00 1 0.000 0 0 2024-04-22T23:03:23.668 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.3 0.00 0.00 1 0.000 0 0 2024-04-22T23:03:23.668 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.3 0.00 0.00 1 0.000 0 0 2024-04-22T23:03:23.668 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:03:23.668 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-04-22T23:03:23.668 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T23:03:23.668 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T23:03:23.669 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 165.3 0.00 0.00 1 0.000 0 0 2024-04-22T23:03:23.669 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T23:03:23.669 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T23:03:23.669 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T23:03:23.669 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T23:03:23.670 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T23:03:23.670 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T23:03:23.670 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative compaction: 0.00 GB write, 25.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T23:03:23.670 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval compaction: 0.00 GB write, 25.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T23:03:23.670 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T23:03:23.670 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:03:23.671 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T23:03:23.671 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:03:23.671 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-04-22T23:03:23.671 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T23:03:23.671 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T23:03:23.671 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.3 0.00 0.00 1 0.000 0 0 2024-04-22T23:03:23.672 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.3 0.00 0.00 1 0.000 0 0 2024-04-22T23:03:23.672 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-22T23:03:23.672 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:03:23.672 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-04-22T23:03:23.672 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T23:03:23.672 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T23:03:23.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 165.3 0.00 0.00 1 0.000 0 0 2024-04-22T23:03:23.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T23:03:23.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T23:03:23.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T23:03:23.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T23:03:23.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T23:03:23.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T23:03:23.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative compaction: 0.00 GB write, 25.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T23:03:23.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T23:03:23.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T23:03:23.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:03:23.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T23:03:23.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: starting mon.smithi069 rank 0 at public addrs [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] at bind addrs [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi069 fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:03:23.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: mon.smithi069@-1(???) e1 preinit fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:03:23.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: mon.smithi069@-1(???).mds e1 new map 2024-04-22T23:03:23.675 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: mon.smithi069@-1(???).mds e1 print_map 2024-04-22T23:03:23.675 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: e1 2024-04-22T23:03:23.675 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T23:03:23.675 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T23:03:23.675 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: legacy client fscid: -1 2024-04-22T23:03:23.675 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:03:23.675 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: No filesystems configured 2024-04-22T23:03:23.676 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: mon.smithi069@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-22T23:03:23.676 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: mon.smithi069@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T23:03:23.676 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: mon.smithi069@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T23:03:23.676 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: mon.smithi069@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T23:03:23.676 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: mon.smithi069@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-22T23:03:23.676 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 bash[21840]: 008ad8fe38c6996b05863941b6022e3e37280cf81fb93b1d02ce3e90d6e4800d 2024-04-22T23:03:23.676 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 systemd[1]: Started Ceph mon.smithi069 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:03:23.677 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: mon.smithi069 is new leader, mons smithi069 in quorum (ranks 0) 2024-04-22T23:03:23.677 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: monmap e1: 1 mons at {smithi069=[v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0]} 2024-04-22T23:03:23.677 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: fsmap 2024-04-22T23:03:23.677 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T23:03:23.677 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:23 smithi069 ceph-mon[21859]: mgrmap e1: no daemons active 2024-04-22T23:03:24.515 INFO:teuthology.orchestra.run.smithi069.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-22T23:03:24.516 INFO:teuthology.orchestra.run.smithi069.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-22T23:03:24.516 INFO:teuthology.orchestra.run.smithi069.stderr:Creating mgr... 2024-04-22T23:03:24.516 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying port 9283 ... 2024-04-22T23:03:24.685 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Failed to reset failed state of unit ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mgr.smithi069.lyiiqg.service: Unit ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mgr.smithi069.lyiiqg.service not loaded. 2024-04-22T23:03:24.692 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/ceph-6928ed7e-00fc-11ef-bc93-c7b262605968.target.wants/ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mgr.smithi069.lyiiqg.service → /etc/systemd/system/ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@.service. 2024-04-22T23:03:25.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:24 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3635413560' entity='client.admin' 2024-04-22T23:03:25.403 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: disabled 2024-04-22T23:03:25.410 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-04-22T23:03:25.410 INFO:teuthology.orchestra.run.smithi069.stderr:firewalld.service is not enabled 2024-04-22T23:03:25.410 INFO:teuthology.orchestra.run.smithi069.stderr:Not possible to enable service . firewalld.service is not available 2024-04-22T23:03:25.418 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: disabled 2024-04-22T23:03:25.425 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-04-22T23:03:25.426 INFO:teuthology.orchestra.run.smithi069.stderr:firewalld.service is not enabled 2024-04-22T23:03:25.426 INFO:teuthology.orchestra.run.smithi069.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-22T23:03:25.426 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mgr to start... 2024-04-22T23:03:25.426 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mgr... 2024-04-22T23:03:25.963 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:25.963 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-22T23:03:25.963 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsid": "6928ed7e-00fc-11ef-bc93-c7b262605968", 2024-04-22T23:03:25.963 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "health": { 2024-04-22T23:03:25.963 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T23:03:25.963 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T23:03:25.963 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T23:03:25.963 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:25.963 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T23:03:25.963 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T23:03:25.963 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 0 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "smithi069" 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:25.964 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T23:03:25.965 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T23:03:25.965 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T23:03:25.965 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T23:03:25.969 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T23:03:25.969 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T23:03:25.969 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:25.969 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T23:03:25.969 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T23:03:25.969 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:25.970 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": false, 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "iostat", 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "nfs", 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "restful" 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modified": "2024-04-22T23:03:17.786637+0000", 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-22T23:03:25.971 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:25.972 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T23:03:25.972 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-22T23:03:26.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:25 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2676638881' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T23:03:26.532 INFO:teuthology.orchestra.run.smithi069.stderr:mgr not available, waiting (1/15)... 2024-04-22T23:03:29.125 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:29.125 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-22T23:03:29.125 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsid": "6928ed7e-00fc-11ef-bc93-c7b262605968", 2024-04-22T23:03:29.125 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "health": { 2024-04-22T23:03:29.125 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T23:03:29.125 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 0 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "smithi069" 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:29.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T23:03:29.127 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": false, 2024-04-22T23:03:29.128 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "iostat", 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "nfs", 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "restful" 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modified": "2024-04-22T23:03:17.786637+0000", 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T23:03:29.129 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-22T23:03:29.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:29 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1434564787' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T23:03:29.706 INFO:teuthology.orchestra.run.smithi069.stderr:mgr not available, waiting (2/15)... 2024-04-22T23:03:31.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: Activating manager daemon smithi069.lyiiqg 2024-04-22T23:03:31.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: mgrmap e2: smithi069.lyiiqg(active, starting, since 0.00291623s) 2024-04-22T23:03:31.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: from='mgr.14100 172.21.15.69:0/1040039171' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T23:03:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: from='mgr.14100 172.21.15.69:0/1040039171' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T23:03:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: from='mgr.14100 172.21.15.69:0/1040039171' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T23:03:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: from='mgr.14100 172.21.15.69:0/1040039171' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:03:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: from='mgr.14100 172.21.15.69:0/1040039171' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi069.lyiiqg", "id": "smithi069.lyiiqg"}]: dispatch 2024-04-22T23:03:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: Manager daemon smithi069.lyiiqg is now available 2024-04-22T23:03:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: from='mgr.14100 172.21.15.69:0/1040039171' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:03:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: from='mgr.14100 172.21.15.69:0/1040039171' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:03:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: from='mgr.14100 172.21.15.69:0/1040039171' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: from='mgr.14100 172.21.15.69:0/1040039171' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:31 smithi069 ceph-mon[21859]: from='mgr.14100 172.21.15.69:0/1040039171' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:32.272 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:32.272 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-22T23:03:32.272 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsid": "6928ed7e-00fc-11ef-bc93-c7b262605968", 2024-04-22T23:03:32.272 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "health": { 2024-04-22T23:03:32.272 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T23:03:32.272 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T23:03:32.272 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T23:03:32.272 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:32.272 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T23:03:32.272 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 0 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "smithi069" 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T23:03:32.273 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T23:03:32.274 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T23:03:32.274 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T23:03:32.274 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T23:03:32.274 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T23:03:32.274 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:32.274 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T23:03:32.274 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T23:03:32.274 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T23:03:32.274 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T23:03:32.274 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T23:03:32.274 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": false, 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T23:03:32.275 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "iostat", 2024-04-22T23:03:32.276 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "nfs", 2024-04-22T23:03:32.276 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "restful" 2024-04-22T23:03:32.276 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-22T23:03:32.276 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-22T23:03:32.276 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:32.276 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T23:03:32.276 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:32.276 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modified": "2024-04-22T23:03:17.786637+0000", 2024-04-22T23:03:32.276 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-22T23:03:32.276 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:32.276 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T23:03:32.276 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-22T23:03:32.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:32 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1688213429' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T23:03:32.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:32 smithi069 ceph-mon[21859]: mgrmap e3: smithi069.lyiiqg(active, since 1.00569s) 2024-04-22T23:03:32.830 INFO:teuthology.orchestra.run.smithi069.stderr:mgr not available, waiting (3/15)... 2024-04-22T23:03:34.548 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:34 smithi069 ceph-mon[21859]: mgrmap e4: smithi069.lyiiqg(active, since 2s) 2024-04-22T23:03:35.562 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:35.562 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsid": "6928ed7e-00fc-11ef-bc93-c7b262605968", 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "health": { 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 0 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "smithi069" 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-22T23:03:35.563 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T23:03:35.564 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:35.567 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T23:03:35.568 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": true, 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "iostat", 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "nfs", 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "restful" 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modified": "2024-04-22T23:03:17.786637+0000", 2024-04-22T23:03:35.569 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-22T23:03:35.570 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-22T23:03:35.570 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T23:03:35.570 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-22T23:03:35.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:35 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3003370433' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T23:03:36.278 INFO:teuthology.orchestra.run.smithi069.stderr:mgr is available 2024-04-22T23:03:36.855 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:36.855 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [global] 2024-04-22T23:03:36.856 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: fsid = 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:03:36.856 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-22T23:03:36.856 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-22T23:03:36.856 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-22T23:03:36.856 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-22T23:03:36.856 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:36.856 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [mgr] 2024-04-22T23:03:36.856 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-22T23:03:36.856 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-22T23:03:36.856 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [osd] 2024-04-22T23:03:36.856 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-22T23:03:36.856 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-22T23:03:37.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:36 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/399525344' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T23:03:37.442 INFO:teuthology.orchestra.run.smithi069.stderr:Enabling cephadm module... 2024-04-22T23:03:38.435 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:38 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1575122453' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-22T23:03:39.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:39 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1575122453' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-22T23:03:39.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:39 smithi069 ceph-mon[21859]: mgrmap e5: smithi069.lyiiqg(active, since 6s) 2024-04-22T23:03:39.449 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-22T23:03:39.449 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-22T23:03:39.449 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": true, 2024-04-22T23:03:39.449 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "active_name": "smithi069.lyiiqg", 2024-04-22T23:03:39.449 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-22T23:03:39.450 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-22T23:03:40.054 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for the mgr to restart... 2024-04-22T23:03:40.054 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mgr epoch 5... 2024-04-22T23:03:40.324 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:40 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2807368282' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T23:03:45.398 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-22T23:03:45.398 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-22T23:03:45.398 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "initialized": true 2024-04-22T23:03:45.398 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-22T23:03:45.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: Active manager daemon smithi069.lyiiqg restarted 2024-04-22T23:03:45.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: Activating manager daemon smithi069.lyiiqg 2024-04-22T23:03:45.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: osdmap e2: 0 total, 0 up, 0 in 2024-04-22T23:03:45.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: mgrmap e6: smithi069.lyiiqg(active, starting, since 0.204183s) 2024-04-22T23:03:45.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:03:45.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi069.lyiiqg", "id": "smithi069.lyiiqg"}]: dispatch 2024-04-22T23:03:45.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T23:03:45.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T23:03:45.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T23:03:45.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: Manager daemon smithi069.lyiiqg is now available 2024-04-22T23:03:45.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:45.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:45.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:03:45.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:03:45.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:03:45.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:03:45.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:45 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:03:46.018 INFO:teuthology.orchestra.run.smithi069.stderr:mgr epoch 5 is available 2024-04-22T23:03:46.019 INFO:teuthology.orchestra.run.smithi069.stderr:Setting orchestrator backend to cephadm... 2024-04-22T23:03:46.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:46 smithi069 ceph-mon[21859]: mgrmap e7: smithi069.lyiiqg(active, since 1.20734s) 2024-04-22T23:03:46.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:46 smithi069 ceph-mon[21859]: [22/Apr/2024:23:03:45] ENGINE Bus STARTING 2024-04-22T23:03:46.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:46 smithi069 ceph-mon[21859]: [22/Apr/2024:23:03:45] ENGINE Serving on https://172.21.15.69:7150 2024-04-22T23:03:46.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:46 smithi069 ceph-mon[21859]: [22/Apr/2024:23:03:45] ENGINE Bus STARTED 2024-04-22T23:03:46.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:46 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:46.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:46 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:03:47.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:47 smithi069 ceph-mon[21859]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:03:47.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:47 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:47.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:47 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:03:47.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:47 smithi069 ceph-mon[21859]: mgrmap e8: smithi069.lyiiqg(active, since 2s) 2024-04-22T23:03:47.953 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: value unchanged 2024-04-22T23:03:48.514 INFO:teuthology.orchestra.run.smithi069.stderr:Generating ssh key... 2024-04-22T23:03:48.857 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:48 smithi069 ceph-mon[21859]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:03:49.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:49 smithi069 ceph-mon[21859]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:03:49.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:49 smithi069 ceph-mon[21859]: Generating ssh key... 2024-04-22T23:03:49.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:49 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:49.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:49 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:50.583 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDa3QzyegYzJlrOXIUiG7AAYEtYn4j4gBU3utl5OWm6foxV1Fr5Hbh9bt7bJ11f5FG41r+LaNZCUAe04baW0HBq5Fpz/dEqp9Vm4CLS8L4aSczJhLyDFrO5tXWqzRE57EQSWqJrvppnbgCjj4kAufFcAUrq/uTBjI+CvnX+/ROr2KuzaVq+9REoyObZ3gxTbjNLEEUCi2ZclRl0bs/WEgmR6hPCcJIz0w5vc4p4PPTEaqFfiNv1tcysX9LSegtdev7ouWBpxM/ylShynR90x+tqqCOUNy2qcOy+aCNGSFs+mJwz9inWtwOw5JPYeKQ7zHgi0vlVp4enx2ntqMTwqe/IBUryNRVJWiarQ9SdTCX6hBA93vxNerrIzQEwtMiSdACMMwam3zljfBf3+aHWM9W8OmJOnP0QF2b/+ByDo7KlquutHxXQTM/xPeiAx9kAyJ0qgmiMzwT4SI9F9Sl/dZ7SDbqMeTeDHqySW5dJ1EtmQND6rDObrmktO69XWQkNxfU= ceph-6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:03:51.168 INFO:teuthology.orchestra.run.smithi069.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-22T23:03:51.168 INFO:teuthology.orchestra.run.smithi069.stderr:Adding key to root@localhost authorized_keys... 2024-04-22T23:03:51.168 INFO:teuthology.orchestra.run.smithi069.stderr:Adding host smithi069... 2024-04-22T23:03:51.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:51 smithi069 ceph-mon[21859]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:03:52.386 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:52 smithi069 ceph-mon[21859]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi069", "addr": "172.21.15.69", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:03:52.386 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:52 smithi069 ceph-mon[21859]: Deploying cephadm binary to smithi069 2024-04-22T23:03:52.616 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Added host 'smithi069' with addr '172.21.15.69' 2024-04-22T23:03:53.182 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying mon service with default placement... 2024-04-22T23:03:53.826 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:53 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:53.826 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:53 smithi069 ceph-mon[21859]: Added host smithi069 2024-04-22T23:03:53.826 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:53 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:03:53.826 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-22T23:03:54.541 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying mgr service with default placement... 2024-04-22T23:03:55.135 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:54 smithi069 ceph-mon[21859]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:03:55.135 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:54 smithi069 ceph-mon[21859]: Saving service mon spec with placement count:5 2024-04-22T23:03:55.135 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:54 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:55.169 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-22T23:03:56.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:56 smithi069 ceph-mon[21859]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:03:56.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:56 smithi069 ceph-mon[21859]: Saving service mgr spec with placement count:2 2024-04-22T23:03:56.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:56 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:56.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:56 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:56.645 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying crash service with default placement... 2024-04-22T23:03:57.288 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled crash update... 2024-04-22T23:03:57.913 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying prometheus service with default placement... 2024-04-22T23:03:58.577 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:58 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:58.577 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:58 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:58.577 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-04-22T23:03:59.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:59 smithi069 ceph-mon[21859]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:03:59.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:59 smithi069 ceph-mon[21859]: Saving service crash spec with placement * 2024-04-22T23:03:59.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:03:59 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:03:59.271 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying grafana service with default placement... 2024-04-22T23:03:59.888 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-04-22T23:04:00.493 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:00 smithi069 ceph-mon[21859]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:04:00.493 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:00 smithi069 ceph-mon[21859]: Saving service prometheus spec with placement count:1 2024-04-22T23:04:00.493 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:00 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:00.495 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying node-exporter service with default placement... 2024-04-22T23:04:01.118 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-04-22T23:04:01.373 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:01 smithi069 ceph-mon[21859]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:04:01.373 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:01 smithi069 ceph-mon[21859]: Saving service grafana spec with placement count:1 2024-04-22T23:04:01.373 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:01 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:01.373 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:01 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:01.744 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying alertmanager service with default placement... 2024-04-22T23:04:02.402 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-04-22T23:04:02.424 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:02 smithi069 ceph-mon[21859]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:04:02.424 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:02 smithi069 ceph-mon[21859]: Saving service node-exporter spec with placement * 2024-04-22T23:04:02.424 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:02 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:03.630 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:03 smithi069 ceph-mon[21859]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:04:03.630 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:03 smithi069 ceph-mon[21859]: Saving service alertmanager spec with placement count:1 2024-04-22T23:04:03.630 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:03 smithi069 ceph-mon[21859]: from='mgr.14120 172.21.15.69:0/2427246872' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:05.801 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:05 smithi069 ceph-mon[21859]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:05.801 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:05 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3620423845' entity='client.admin' 2024-04-22T23:04:06.404 INFO:teuthology.orchestra.run.smithi069.stderr:Enabling the dashboard module... 2024-04-22T23:04:07.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:06 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2431971282' entity='client.admin' 2024-04-22T23:04:08.089 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:07 smithi069 ceph-mon[21859]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:08.089 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:07 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2404885352' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-22T23:04:09.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:08 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2404885352' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-22T23:04:09.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:08 smithi069 ceph-mon[21859]: mgrmap e9: smithi069.lyiiqg(active, since 23s) 2024-04-22T23:04:09.192 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-22T23:04:09.192 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-22T23:04:09.192 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": true, 2024-04-22T23:04:09.192 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "active_name": "smithi069.lyiiqg", 2024-04-22T23:04:09.192 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-22T23:04:09.192 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-22T23:04:09.800 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for the mgr to restart... 2024-04-22T23:04:09.800 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mgr epoch 9... 2024-04-22T23:04:10.082 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:09 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2128833406' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T23:04:14.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:14 smithi069 ceph-mon[21859]: Active manager daemon smithi069.lyiiqg restarted 2024-04-22T23:04:14.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:14 smithi069 ceph-mon[21859]: Activating manager daemon smithi069.lyiiqg 2024-04-22T23:04:14.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:14 smithi069 ceph-mon[21859]: osdmap e3: 0 total, 0 up, 0 in 2024-04-22T23:04:15.406 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-22T23:04:15.407 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-22T23:04:15.407 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "initialized": true 2024-04-22T23:04:15.407 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-22T23:04:15.807 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: mgrmap e10: smithi069.lyiiqg(active, starting, since 0.550607s) 2024-04-22T23:04:15.807 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:04:15.807 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi069.lyiiqg", "id": "smithi069.lyiiqg"}]: dispatch 2024-04-22T23:04:15.807 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T23:04:15.808 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T23:04:15.808 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T23:04:15.808 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: Manager daemon smithi069.lyiiqg is now available 2024-04-22T23:04:15.808 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:04:15.808 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:04:15.808 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:04:15.808 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:04:15.808 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: [22/Apr/2024:23:04:14] ENGINE Bus STARTING 2024-04-22T23:04:15.808 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: [22/Apr/2024:23:04:14] ENGINE Serving on https://172.21.15.69:7150 2024-04-22T23:04:15.808 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: [22/Apr/2024:23:04:14] ENGINE Bus STARTED 2024-04-22T23:04:15.808 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:15 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:16.094 INFO:teuthology.orchestra.run.smithi069.stderr:mgr epoch 9 is available 2024-04-22T23:04:16.094 INFO:teuthology.orchestra.run.smithi069.stderr:Generating a dashboard self-signed certificate... 2024-04-22T23:04:16.615 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:16 smithi069 ceph-mon[21859]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-22T23:04:16.615 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:16 smithi069 ceph-mon[21859]: mgrmap e11: smithi069.lyiiqg(active, since 1.55281s) 2024-04-22T23:04:16.615 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:16 smithi069 ceph-mon[21859]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-22T23:04:16.763 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-22T23:04:17.453 INFO:teuthology.orchestra.run.smithi069.stderr:Creating initial admin user... 2024-04-22T23:04:17.655 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:17 smithi069 ceph-mon[21859]: mgrmap e12: smithi069.lyiiqg(active, since 2s) 2024-04-22T23:04:17.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:17 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:17.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:17 smithi069 ceph-mon[21859]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:04:17.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:17 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:17.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:17 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:18.527 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$4Qaw3tA4u9ek2365XzozWOl/tpkypyu2El1WPkvlZM.YkTLmUvfsi", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1713827058, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-22T23:04:19.247 INFO:teuthology.orchestra.run.smithi069.stderr:Fetching dashboard port number... 2024-04-22T23:04:19.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:19 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:19.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:19 smithi069 ceph-mon[21859]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:04:19.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:19 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:20.000 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 8443 2024-04-22T23:04:20.538 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:20 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/58227787' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-22T23:04:21.303 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:21 smithi069 ceph-mon[21859]: mgrmap e13: smithi069.lyiiqg(active, since 6s) 2024-04-22T23:04:21.315 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: disabled 2024-04-22T23:04:21.323 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-04-22T23:04:21.323 INFO:teuthology.orchestra.run.smithi069.stderr:firewalld.service is not enabled 2024-04-22T23:04:21.323 INFO:teuthology.orchestra.run.smithi069.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-22T23:04:21.325 INFO:teuthology.orchestra.run.smithi069.stderr:Ceph Dashboard is now available at: 2024-04-22T23:04:21.325 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-22T23:04:21.325 INFO:teuthology.orchestra.run.smithi069.stderr: URL: https://smithi069:8443/ 2024-04-22T23:04:21.325 INFO:teuthology.orchestra.run.smithi069.stderr: User: admin 2024-04-22T23:04:21.325 INFO:teuthology.orchestra.run.smithi069.stderr: Password: abnlu8amav 2024-04-22T23:04:21.326 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-22T23:04:21.326 INFO:teuthology.orchestra.run.smithi069.stderr:Enabling autotune for osd_memory_target 2024-04-22T23:04:23.431 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-22T23:04:24.070 INFO:teuthology.orchestra.run.smithi069.stderr:You can access the Ceph CLI with: 2024-04-22T23:04:24.071 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-22T23:04:24.071 INFO:teuthology.orchestra.run.smithi069.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-22T23:04:24.071 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-22T23:04:24.071 INFO:teuthology.orchestra.run.smithi069.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-22T23:04:24.071 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-22T23:04:24.071 INFO:teuthology.orchestra.run.smithi069.stderr: ceph telemetry on 2024-04-22T23:04:24.071 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-22T23:04:24.071 INFO:teuthology.orchestra.run.smithi069.stderr:For more information see: 2024-04-22T23:04:24.071 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-22T23:04:24.071 INFO:teuthology.orchestra.run.smithi069.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-22T23:04:24.071 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-22T23:04:24.071 INFO:teuthology.orchestra.run.smithi069.stderr:Bootstrap complete. 2024-04-22T23:04:24.105 INFO:tasks.cephadm:Fetching config... 2024-04-22T23:04:24.106 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-22T23:04:24.106 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-22T23:04:24.122 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-22T23:04:24.122 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-22T23:04:24.122 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-22T23:04:24.177 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-22T23:04:24.177 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-22T23:04:24.177 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/keyring of=/dev/stdout 2024-04-22T23:04:24.240 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-22T23:04:24.240 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-22T23:04:24.240 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-22T23:04:24.295 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-22T23:04:24.295 DEBUG:teuthology.orchestra.run.smithi069:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDa3QzyegYzJlrOXIUiG7AAYEtYn4j4gBU3utl5OWm6foxV1Fr5Hbh9bt7bJ11f5FG41r+LaNZCUAe04baW0HBq5Fpz/dEqp9Vm4CLS8L4aSczJhLyDFrO5tXWqzRE57EQSWqJrvppnbgCjj4kAufFcAUrq/uTBjI+CvnX+/ROr2KuzaVq+9REoyObZ3gxTbjNLEEUCi2ZclRl0bs/WEgmR6hPCcJIz0w5vc4p4PPTEaqFfiNv1tcysX9LSegtdev7ouWBpxM/ylShynR90x+tqqCOUNy2qcOy+aCNGSFs+mJwz9inWtwOw5JPYeKQ7zHgi0vlVp4enx2ntqMTwqe/IBUryNRVJWiarQ9SdTCX6hBA93vxNerrIzQEwtMiSdACMMwam3zljfBf3+aHWM9W8OmJOnP0QF2b/+ByDo7KlquutHxXQTM/xPeiAx9kAyJ0qgmiMzwT4SI9F9Sl/dZ7SDbqMeTeDHqySW5dJ1EtmQND6rDObrmktO69XWQkNxfU= ceph-6928ed7e-00fc-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-22T23:04:24.367 INFO:teuthology.orchestra.run.smithi069.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDa3QzyegYzJlrOXIUiG7AAYEtYn4j4gBU3utl5OWm6foxV1Fr5Hbh9bt7bJ11f5FG41r+LaNZCUAe04baW0HBq5Fpz/dEqp9Vm4CLS8L4aSczJhLyDFrO5tXWqzRE57EQSWqJrvppnbgCjj4kAufFcAUrq/uTBjI+CvnX+/ROr2KuzaVq+9REoyObZ3gxTbjNLEEUCi2ZclRl0bs/WEgmR6hPCcJIz0w5vc4p4PPTEaqFfiNv1tcysX9LSegtdev7ouWBpxM/ylShynR90x+tqqCOUNy2qcOy+aCNGSFs+mJwz9inWtwOw5JPYeKQ7zHgi0vlVp4enx2ntqMTwqe/IBUryNRVJWiarQ9SdTCX6hBA93vxNerrIzQEwtMiSdACMMwam3zljfBf3+aHWM9W8OmJOnP0QF2b/+ByDo7KlquutHxXQTM/xPeiAx9kAyJ0qgmiMzwT4SI9F9Sl/dZ7SDbqMeTeDHqySW5dJ1EtmQND6rDObrmktO69XWQkNxfU= ceph-6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:04:24.379 DEBUG:teuthology.orchestra.run.smithi149:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDa3QzyegYzJlrOXIUiG7AAYEtYn4j4gBU3utl5OWm6foxV1Fr5Hbh9bt7bJ11f5FG41r+LaNZCUAe04baW0HBq5Fpz/dEqp9Vm4CLS8L4aSczJhLyDFrO5tXWqzRE57EQSWqJrvppnbgCjj4kAufFcAUrq/uTBjI+CvnX+/ROr2KuzaVq+9REoyObZ3gxTbjNLEEUCi2ZclRl0bs/WEgmR6hPCcJIz0w5vc4p4PPTEaqFfiNv1tcysX9LSegtdev7ouWBpxM/ylShynR90x+tqqCOUNy2qcOy+aCNGSFs+mJwz9inWtwOw5JPYeKQ7zHgi0vlVp4enx2ntqMTwqe/IBUryNRVJWiarQ9SdTCX6hBA93vxNerrIzQEwtMiSdACMMwam3zljfBf3+aHWM9W8OmJOnP0QF2b/+ByDo7KlquutHxXQTM/xPeiAx9kAyJ0qgmiMzwT4SI9F9Sl/dZ7SDbqMeTeDHqySW5dJ1EtmQND6rDObrmktO69XWQkNxfU= ceph-6928ed7e-00fc-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-22T23:04:24.419 INFO:teuthology.orchestra.run.smithi149.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDa3QzyegYzJlrOXIUiG7AAYEtYn4j4gBU3utl5OWm6foxV1Fr5Hbh9bt7bJ11f5FG41r+LaNZCUAe04baW0HBq5Fpz/dEqp9Vm4CLS8L4aSczJhLyDFrO5tXWqzRE57EQSWqJrvppnbgCjj4kAufFcAUrq/uTBjI+CvnX+/ROr2KuzaVq+9REoyObZ3gxTbjNLEEUCi2ZclRl0bs/WEgmR6hPCcJIz0w5vc4p4PPTEaqFfiNv1tcysX9LSegtdev7ouWBpxM/ylShynR90x+tqqCOUNy2qcOy+aCNGSFs+mJwz9inWtwOw5JPYeKQ7zHgi0vlVp4enx2ntqMTwqe/IBUryNRVJWiarQ9SdTCX6hBA93vxNerrIzQEwtMiSdACMMwam3zljfBf3+aHWM9W8OmJOnP0QF2b/+ByDo7KlquutHxXQTM/xPeiAx9kAyJ0qgmiMzwT4SI9F9Sl/dZ7SDbqMeTeDHqySW5dJ1EtmQND6rDObrmktO69XWQkNxfU= ceph-6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:04:24.431 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-22T23:04:24.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:24 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3565874055' entity='client.admin' 2024-04-22T23:04:27.731 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-22T23:04:27.731 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-22T23:04:28.218 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:28 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3268602202' entity='client.admin' 2024-04-22T23:04:30.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:30 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:30.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:30 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd/host:smithi069", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:04:30.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:30 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:30.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:30 smithi069 ceph-mon[21859]: Deploying daemon alertmanager.smithi069 on smithi069 2024-04-22T23:04:31.193 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi149 2024-04-22T23:04:31.193 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2024-04-22T23:04:31.193 DEBUG:teuthology.orchestra.run.smithi149:> dd of=/etc/ceph/ceph.conf 2024-04-22T23:04:31.212 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2024-04-22T23:04:31.212 DEBUG:teuthology.orchestra.run.smithi149:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:04:31.270 INFO:tasks.cephadm:Adding host smithi149 to orchestrator... 2024-04-22T23:04:31.270 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch host add smithi149 2024-04-22T23:04:31.581 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:31 smithi069 ceph-mon[21859]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:04:31.581 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:31 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:35.417 INFO:teuthology.orchestra.run.smithi069.stdout:Added host 'smithi149' with addr '172.21.15.149' 2024-04-22T23:04:35.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:35 smithi069 ceph-mon[21859]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:35.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:35 smithi069 ceph-mon[21859]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi149", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:04:35.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:35 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:36.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:36 smithi069 ceph-mon[21859]: Deploying cephadm binary to smithi149 2024-04-22T23:04:36.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:36 smithi069 ceph-mon[21859]: Added host smithi149 2024-04-22T23:04:36.835 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-22T23:04:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:37 smithi069 ceph-mon[21859]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:39.556 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:39 smithi069 ceph-mon[21859]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:39.556 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:04:39.556 INFO:teuthology.orchestra.run.smithi069.stdout:[{"addr": "172.21.15.69", "hostname": "smithi069", "labels": [], "status": ""}, {"addr": "172.21.15.149", "hostname": "smithi149", "labels": [], "status": ""}] 2024-04-22T23:04:40.569 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:40 smithi069 ceph-mon[21859]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:04:40.569 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:40 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:40.570 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:40 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:04:40.570 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:40 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-22T23:04:40.570 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:40 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:04:40.664 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-22T23:04:40.665 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-22T23:04:41.613 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:41 smithi069 ceph-mon[21859]: Deploying daemon crash.smithi069 on smithi069 2024-04-22T23:04:41.613 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:41 smithi069 ceph-mon[21859]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:43.570 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:43 smithi069 ceph-mon[21859]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:44.463 INFO:teuthology.orchestra.run.smithi069.stderr:adjusted tunables profile to default 2024-04-22T23:04:44.817 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:44 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3768168311' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-22T23:04:44.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:44 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:44.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:44 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:44.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:44 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:44.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:44 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T23:04:44.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:44 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:44.819 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:44 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:45.118 INFO:tasks.cephadm:Adding mon.smithi069 on smithi069 2024-04-22T23:04:45.118 INFO:tasks.cephadm:Adding mon.smithi149 on smithi149 2024-04-22T23:04:45.118 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch apply mon '2;smithi069:172.21.15.69=smithi069;smithi149:172.21.15.149=smithi149' 2024-04-22T23:04:45.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:45 smithi069 ceph-mon[21859]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:45.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:45 smithi069 ceph-mon[21859]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T23:04:45.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:45 smithi069 ceph-mon[21859]: Deploying daemon grafana.smithi069 on smithi069 2024-04-22T23:04:45.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:45 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3768168311' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-22T23:04:45.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:45 smithi069 ceph-mon[21859]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T23:04:47.228 INFO:teuthology.orchestra.run.smithi149.stdout:Scheduled mon update... 2024-04-22T23:04:47.775 DEBUG:teuthology.orchestra.run.smithi149:mon.smithi149> sudo journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi149.service 2024-04-22T23:04:47.777 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T23:04:47.778 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T23:04:47.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:47 smithi069 ceph-mon[21859]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:47.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:47 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:48.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:48 smithi069 ceph-mon[21859]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi069:172.21.15.69=smithi069;smithi149:172.21.15.149=smithi149", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:04:48.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:48 smithi069 ceph-mon[21859]: Saving service mon spec with placement smithi069:172.21.15.69=smithi069;smithi149:172.21.15.149=smithi149;count:2 2024-04-22T23:04:49.644 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T23:04:49.644 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","modified":"2024-04-22T23:03:14.830052Z","created":"2024-04-22T23:03:14.830052Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T23:04:49.646 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-04-22T23:04:49.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:49 smithi069 ceph-mon[21859]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:50.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:50 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/4132147073' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T23:04:51.204 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T23:04:51.205 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T23:04:51.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:51 smithi069 ceph-mon[21859]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:53.000 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T23:04:53.000 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","modified":"2024-04-22T23:03:14.830052Z","created":"2024-04-22T23:03:14.830052Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T23:04:53.002 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-04-22T23:04:53.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:53 smithi069 ceph-mon[21859]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:53.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:53 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/3349620356' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T23:04:54.641 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T23:04:54.642 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T23:04:55.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:55 smithi069 ceph-mon[21859]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:56.632 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T23:04:56.632 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","modified":"2024-04-22T23:03:14.830052Z","created":"2024-04-22T23:03:14.830052Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T23:04:56.634 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-04-22T23:04:57.636 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:57 smithi069 ceph-mon[21859]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:57.636 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:57 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/2852858199' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T23:04:57.636 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:57 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:04:58.235 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T23:04:58.235 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T23:04:58.638 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:58 smithi069 ceph-mon[21859]: Deploying daemon node-exporter.smithi069 on smithi069 2024-04-22T23:04:59.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:59 smithi069 ceph-mon[21859]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:04:59.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:04:59 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:00.046 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T23:05:00.046 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","modified":"2024-04-22T23:03:14.830052Z","created":"2024-04-22T23:03:14.830052Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T23:05:00.048 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-04-22T23:05:00.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:00 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/1203255662' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T23:05:00.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:00 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:00.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:00 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-22T23:05:01.544 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:01 smithi069 ceph-mon[21859]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:01.544 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:01 smithi069 ceph-mon[21859]: from='mgr.14162 172.21.15.69:0/3587253018' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-22T23:05:01.544 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:01 smithi069 ceph-mon[21859]: mgrmap e14: smithi069.lyiiqg(active, since 47s) 2024-04-22T23:05:01.603 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T23:05:01.603 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T23:05:03.461 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T23:05:03.461 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","modified":"2024-04-22T23:03:14.830052Z","created":"2024-04-22T23:03:14.830052Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T23:05:03.463 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-04-22T23:05:04.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:04 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/2835279835' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T23:05:04.984 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T23:05:04.984 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T23:05:06.911 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T23:05:06.911 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","modified":"2024-04-22T23:03:14.830052Z","created":"2024-04-22T23:03:14.830052Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T23:05:06.913 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-04-22T23:05:07.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:06 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/1795948709' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T23:05:08.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:07 smithi069 ceph-mon[21859]: Active manager daemon smithi069.lyiiqg restarted 2024-04-22T23:05:08.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:07 smithi069 ceph-mon[21859]: Activating manager daemon smithi069.lyiiqg 2024-04-22T23:05:08.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:07 smithi069 ceph-mon[21859]: osdmap e5: 0 total, 0 up, 0 in 2024-04-22T23:05:08.530 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T23:05:08.530 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T23:05:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: mgrmap e15: smithi069.lyiiqg(active, starting, since 0.743871s) 2024-04-22T23:05:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:05:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi069.lyiiqg", "id": "smithi069.lyiiqg"}]: dispatch 2024-04-22T23:05:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T23:05:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T23:05:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T23:05:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: Manager daemon smithi069.lyiiqg is now available 2024-04-22T23:05:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:05:09.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:05:09.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:09.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:05:09.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:09.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:05:09.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:05:09.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:09.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:09.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:10.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:09 smithi069 ceph-mon[21859]: [22/Apr/2024:23:05:08] ENGINE Bus STARTING 2024-04-22T23:05:10.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:09 smithi069 ceph-mon[21859]: [22/Apr/2024:23:05:08] ENGINE Serving on https://172.21.15.69:7150 2024-04-22T23:05:10.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:09 smithi069 ceph-mon[21859]: [22/Apr/2024:23:05:08] ENGINE Bus STARTED 2024-04-22T23:05:10.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:09 smithi069 ceph-mon[21859]: mgrmap e16: smithi069.lyiiqg(active, since 1.7467s) 2024-04-22T23:05:10.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:09 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:10.440 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T23:05:10.440 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","modified":"2024-04-22T23:03:14.830052Z","created":"2024-04-22T23:03:14.830052Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T23:05:10.442 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-04-22T23:05:11.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:10 smithi069 ceph-mon[21859]: mgrmap e17: smithi069.lyiiqg(active, since 2s) 2024-04-22T23:05:11.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:10 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/2397095761' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T23:05:12.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:12 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:12.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:12 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:12.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:12 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd/host:smithi069", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:05:12.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:12 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:12.761 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T23:05:12.761 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T23:05:13.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:13 smithi069 ceph-mon[21859]: Updating smithi069:/etc/ceph/ceph.conf 2024-04-22T23:05:13.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:13 smithi069 ceph-mon[21859]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:05:14.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:14.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:14.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:05:14.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:14.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:05:14.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-22T23:05:14.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:14.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:14 smithi069 ceph-mon[21859]: Deploying daemon crash.smithi149 on smithi149 2024-04-22T23:05:15.159 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T23:05:15.159 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","modified":"2024-04-22T23:03:14.830052Z","created":"2024-04-22T23:03:14.830052Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T23:05:15.161 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-04-22T23:05:15.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:15 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/3250022236' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T23:05:17.579 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T23:05:17.580 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T23:05:18.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:18 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:18.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:18 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.rxqkqk", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:05:18.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:18 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.rxqkqk", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-22T23:05:18.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:18 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:05:18.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:18 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:18.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:18 smithi069 ceph-mon[21859]: Deploying daemon mgr.smithi149.rxqkqk on smithi149 2024-04-22T23:05:18.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:18 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:20.159 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T23:05:20.160 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","modified":"2024-04-22T23:03:14.830052Z","created":"2024-04-22T23:03:14.830052Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T23:05:20.161 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-04-22T23:05:20.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:20 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/1334074995' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T23:05:21.991 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T23:05:21.992 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T23:05:22.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:22.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:05:22.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:22.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:22 smithi069 ceph-mon[21859]: Deploying daemon mon.smithi149 on smithi149 2024-04-22T23:05:24.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:24 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:24.482 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T23:05:24.482 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","modified":"2024-04-22T23:03:14.830052Z","created":"2024-04-22T23:03:14.830052Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T23:05:24.483 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2024-04-22T23:05:25.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:25 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/2760309258' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T23:05:26.860 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-22T23:05:26.860 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T23:05:31.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:05:31.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: mon.smithi069 calling monitor election 2024-04-22T23:05:31.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:05:31.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:05:31.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/3901914465' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/crt"}]: dispatch 2024-04-22T23:05:31.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:31.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:05:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: mon.smithi149 calling monitor election 2024-04-22T23:05:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:05:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:05:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:05:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: mon.smithi069 is new leader, mons smithi069,smithi149 in quorum (ranks 0,1) 2024-04-22T23:05:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: monmap e2: 2 mons at {smithi069=[v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0],smithi149=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0]} 2024-04-22T23:05:31.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: fsmap 2024-04-22T23:05:31.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: osdmap e5: 0 total, 0 up, 0 in 2024-04-22T23:05:31.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: mgrmap e17: smithi069.lyiiqg(active, since 24s) 2024-04-22T23:05:31.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: Standby manager daemon smithi149.rxqkqk started 2024-04-22T23:05:31.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/3901914465' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:05:31.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: overall HEALTH_OK 2024-04-22T23:05:31.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/3901914465' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/key"}]: dispatch 2024-04-22T23:05:31.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/3901914465' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:05:31.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:31 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:31.757 INFO:teuthology.orchestra.run.smithi149.stdout: 2024-04-22T23:05:31.757 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":2,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","modified":"2024-04-22T23:05:26.301919Z","created":"2024-04-22T23:03:14.830052Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi149","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:3300","nonce":0},{"type":"v1","addr":"172.21.15.149:6789","nonce":0}]},"addr":"172.21.15.149:6789/0","public_addr":"172.21.15.149:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-22T23:05:31.759 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 2 2024-04-22T23:05:32.347 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-22T23:05:32.347 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-22T23:05:32.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:32 smithi069 ceph-mon[21859]: Deploying daemon node-exporter.smithi149 on smithi149 2024-04-22T23:05:32.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:32 smithi069 ceph-mon[21859]: mgrmap e18: smithi069.lyiiqg(active, since 24s), standbys: smithi149.rxqkqk 2024-04-22T23:05:32.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:32 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi149.rxqkqk", "id": "smithi149.rxqkqk"}]: dispatch 2024-04-22T23:05:32.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:32 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/3521000602' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T23:05:32.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:32 smithi069 ceph-mon[21859]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:32.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:32 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:05:34.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:34 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:34.195 INFO:teuthology.orchestra.run.smithi069.stdout:# minimal ceph.conf for 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:05:34.195 INFO:teuthology.orchestra.run.smithi069.stdout:[global] 2024-04-22T23:05:34.195 INFO:teuthology.orchestra.run.smithi069.stdout: fsid = 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:05:34.195 INFO:teuthology.orchestra.run.smithi069.stdout: mon_host = [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] 2024-04-22T23:05:34.818 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-22T23:05:34.819 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-22T23:05:34.819 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/ceph/ceph.conf 2024-04-22T23:05:34.844 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-22T23:05:34.844 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:05:34.908 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2024-04-22T23:05:34.908 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/etc/ceph/ceph.conf 2024-04-22T23:05:34.938 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2024-04-22T23:05:34.938 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:05:35.004 INFO:tasks.cephadm:Deploying OSDs... 2024-04-22T23:05:35.004 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-22T23:05:35.005 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/scratch_devs of=/dev/stdout 2024-04-22T23:05:35.020 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-22T23:05:35.020 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_1 2024-04-22T23:05:35.075 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-22T23:05:35.075 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T23:05:35.075 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-22T23:05:35.075 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T23:05:35.075 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T23:05:35.075 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2024-04-22 23:04:57.579907731 +0000 2024-04-22T23:05:35.075 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2024-04-22 23:02:15.909464899 +0000 2024-04-22T23:05:35.075 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2024-04-22 23:02:15.909464899 +0000 2024-04-22T23:05:35.076 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: 2024-04-22 23:02:15.909464899 +0000 2024-04-22T23:05:35.076 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-22T23:05:35.136 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:35 smithi069 ceph-mon[21859]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:35.136 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:35 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3268616035' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:35.138 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2024-04-22T23:05:35.138 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2024-04-22T23:05:35.138 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000251899 s, 2.0 MB/s 2024-04-22T23:05:35.139 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-22T23:05:35.196 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_2 2024-04-22T23:05:35.252 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-22T23:05:35.252 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T23:05:35.252 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 5h/5d Inode: 683 Links: 1 2024-04-22T23:05:35.252 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T23:05:35.252 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T23:05:35.252 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2024-04-22 23:04:57.579907731 +0000 2024-04-22T23:05:35.252 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2024-04-22 23:02:16.238457506 +0000 2024-04-22T23:05:35.252 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2024-04-22 23:02:16.238457506 +0000 2024-04-22T23:05:35.253 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: 2024-04-22 23:02:16.238457506 +0000 2024-04-22T23:05:35.253 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-22T23:05:35.316 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2024-04-22T23:05:35.316 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2024-04-22T23:05:35.316 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000209184 s, 2.4 MB/s 2024-04-22T23:05:35.318 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-22T23:05:35.375 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_3 2024-04-22T23:05:35.430 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-22T23:05:35.431 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T23:05:35.431 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-04-22T23:05:35.431 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T23:05:35.431 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T23:05:35.431 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2024-04-22 23:04:57.579907731 +0000 2024-04-22T23:05:35.431 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2024-04-22 23:02:16.596449460 +0000 2024-04-22T23:05:35.431 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2024-04-22 23:02:16.596449460 +0000 2024-04-22T23:05:35.431 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: 2024-04-22 23:02:16.596449460 +0000 2024-04-22T23:05:35.431 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-22T23:05:35.493 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2024-04-22T23:05:35.494 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2024-04-22T23:05:35.494 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000200744 s, 2.6 MB/s 2024-04-22T23:05:35.495 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-22T23:05:35.550 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_4 2024-04-22T23:05:35.603 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-22T23:05:35.604 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T23:05:35.604 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-22T23:05:35.604 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T23:05:35.604 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T23:05:35.604 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2024-04-22 23:04:57.580907708 +0000 2024-04-22T23:05:35.604 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2024-04-22 23:02:16.942441684 +0000 2024-04-22T23:05:35.604 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2024-04-22 23:02:16.942441684 +0000 2024-04-22T23:05:35.604 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: 2024-04-22 23:02:16.942441684 +0000 2024-04-22T23:05:35.604 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-22T23:05:35.665 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2024-04-22T23:05:35.665 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2024-04-22T23:05:35.665 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.0002244 s, 2.3 MB/s 2024-04-22T23:05:35.667 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-22T23:05:35.722 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2024-04-22T23:05:35.722 DEBUG:teuthology.orchestra.run.smithi149:> dd if=/scratch_devs of=/dev/stdout 2024-04-22T23:05:35.741 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-22T23:05:35.741 DEBUG:teuthology.orchestra.run.smithi149:> stat /dev/vg_nvme/lv_1 2024-04-22T23:05:35.799 INFO:teuthology.orchestra.run.smithi149.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-22T23:05:35.799 INFO:teuthology.orchestra.run.smithi149.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T23:05:35.799 INFO:teuthology.orchestra.run.smithi149.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-22T23:05:35.799 INFO:teuthology.orchestra.run.smithi149.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T23:05:35.800 INFO:teuthology.orchestra.run.smithi149.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T23:05:35.800 INFO:teuthology.orchestra.run.smithi149.stdout:Access: 2024-04-22 23:05:31.919603373 +0000 2024-04-22T23:05:35.800 INFO:teuthology.orchestra.run.smithi149.stdout:Modify: 2024-04-22 23:02:15.913898338 +0000 2024-04-22T23:05:35.800 INFO:teuthology.orchestra.run.smithi149.stdout:Change: 2024-04-22 23:02:15.913898338 +0000 2024-04-22T23:05:35.800 INFO:teuthology.orchestra.run.smithi149.stdout: Birth: 2024-04-22 23:02:15.913898338 +0000 2024-04-22T23:05:35.800 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-22T23:05:35.866 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records in 2024-04-22T23:05:35.866 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records out 2024-04-22T23:05:35.866 INFO:teuthology.orchestra.run.smithi149.stderr:512 bytes copied, 0.000234298 s, 2.2 MB/s 2024-04-22T23:05:35.867 DEBUG:teuthology.orchestra.run.smithi149:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-22T23:05:35.925 DEBUG:teuthology.orchestra.run.smithi149:> stat /dev/vg_nvme/lv_2 2024-04-22T23:05:35.981 INFO:teuthology.orchestra.run.smithi149.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-22T23:05:35.982 INFO:teuthology.orchestra.run.smithi149.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T23:05:35.982 INFO:teuthology.orchestra.run.smithi149.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-22T23:05:35.982 INFO:teuthology.orchestra.run.smithi149.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T23:05:35.982 INFO:teuthology.orchestra.run.smithi149.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T23:05:35.982 INFO:teuthology.orchestra.run.smithi149.stdout:Access: 2024-04-22 23:05:31.920603353 +0000 2024-04-22T23:05:35.982 INFO:teuthology.orchestra.run.smithi149.stdout:Modify: 2024-04-22 23:02:16.268890905 +0000 2024-04-22T23:05:35.982 INFO:teuthology.orchestra.run.smithi149.stdout:Change: 2024-04-22 23:02:16.268890905 +0000 2024-04-22T23:05:35.982 INFO:teuthology.orchestra.run.smithi149.stdout: Birth: 2024-04-22 23:02:16.268890905 +0000 2024-04-22T23:05:35.982 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-22T23:05:36.046 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records in 2024-04-22T23:05:36.047 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records out 2024-04-22T23:05:36.047 INFO:teuthology.orchestra.run.smithi149.stderr:512 bytes copied, 0.000228275 s, 2.2 MB/s 2024-04-22T23:05:36.048 DEBUG:teuthology.orchestra.run.smithi149:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-22T23:05:36.105 DEBUG:teuthology.orchestra.run.smithi149:> stat /dev/vg_nvme/lv_3 2024-04-22T23:05:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-22T23:05:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T23:05:36.161 INFO:teuthology.orchestra.run.smithi149.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-04-22T23:05:36.161 INFO:teuthology.orchestra.run.smithi149.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T23:05:36.161 INFO:teuthology.orchestra.run.smithi149.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T23:05:36.161 INFO:teuthology.orchestra.run.smithi149.stdout:Access: 2024-04-22 23:05:31.920603353 +0000 2024-04-22T23:05:36.162 INFO:teuthology.orchestra.run.smithi149.stdout:Modify: 2024-04-22 23:02:16.624883451 +0000 2024-04-22T23:05:36.162 INFO:teuthology.orchestra.run.smithi149.stdout:Change: 2024-04-22 23:02:16.624883451 +0000 2024-04-22T23:05:36.162 INFO:teuthology.orchestra.run.smithi149.stdout: Birth: 2024-04-22 23:02:16.624883451 +0000 2024-04-22T23:05:36.162 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-22T23:05:36.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:36 smithi149 ceph-mon[23948]: Deploying daemon prometheus.smithi069 on smithi069 2024-04-22T23:05:36.223 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records in 2024-04-22T23:05:36.223 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records out 2024-04-22T23:05:36.223 INFO:teuthology.orchestra.run.smithi149.stderr:512 bytes copied, 0.000225534 s, 2.3 MB/s 2024-04-22T23:05:36.225 DEBUG:teuthology.orchestra.run.smithi149:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-22T23:05:36.283 DEBUG:teuthology.orchestra.run.smithi149:> stat /dev/vg_nvme/lv_4 2024-04-22T23:05:36.339 INFO:teuthology.orchestra.run.smithi149.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-22T23:05:36.339 INFO:teuthology.orchestra.run.smithi149.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T23:05:36.339 INFO:teuthology.orchestra.run.smithi149.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-04-22T23:05:36.339 INFO:teuthology.orchestra.run.smithi149.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T23:05:36.339 INFO:teuthology.orchestra.run.smithi149.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T23:05:36.339 INFO:teuthology.orchestra.run.smithi149.stdout:Access: 2024-04-22 23:05:31.920603353 +0000 2024-04-22T23:05:36.339 INFO:teuthology.orchestra.run.smithi149.stdout:Modify: 2024-04-22 23:02:16.994875702 +0000 2024-04-22T23:05:36.340 INFO:teuthology.orchestra.run.smithi149.stdout:Change: 2024-04-22 23:02:16.994875702 +0000 2024-04-22T23:05:36.340 INFO:teuthology.orchestra.run.smithi149.stdout: Birth: 2024-04-22 23:02:16.994875702 +0000 2024-04-22T23:05:36.340 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-22T23:05:36.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:36 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:36.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:36 smithi069 ceph-mon[21859]: Deploying daemon prometheus.smithi069 on smithi069 2024-04-22T23:05:36.402 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records in 2024-04-22T23:05:36.402 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records out 2024-04-22T23:05:36.402 INFO:teuthology.orchestra.run.smithi149.stderr:512 bytes copied, 0.000197564 s, 2.6 MB/s 2024-04-22T23:05:36.403 DEBUG:teuthology.orchestra.run.smithi149:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-22T23:05:36.459 INFO:tasks.cephadm:Deploying osd.0 on smithi069 with /dev/vg_nvme/lv_4... 2024-04-22T23:05:36.459 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-22T23:05:37.483 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:37 smithi069 ceph-mon[21859]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:37.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:37 smithi149 ceph-mon[23948]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:39.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:39 smithi069 ceph-mon[21859]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:39.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:39 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:39.255 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-22T23:05:39.255 INFO:teuthology.orchestra.run.smithi069.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-22T23:05:39.255 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10+0 records in 2024-04-22T23:05:39.255 INFO:teuthology.orchestra.run.smithi069.stderr:10+0 records out 2024-04-22T23:05:39.255 INFO:teuthology.orchestra.run.smithi069.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0141048 s, 743 MB/s 2024-04-22T23:05:39.255 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping successful for: 2024-04-22T23:05:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:39 smithi149 ceph-mon[23948]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:39 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:41.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:41 smithi069 ceph-mon[21859]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:41 smithi149 ceph-mon[23948]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:41.633 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi069:vg_nvme/lv_4 2024-04-22T23:05:43.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:43 smithi069 ceph-mon[21859]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:43 smithi149 ceph-mon[23948]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:44.461 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:44 smithi069 ceph-mon[21859]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:44.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:44 smithi149 ceph-mon[23948]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:45.686 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:45 smithi069 ceph-mon[21859]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:05:45.686 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:05:45.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:05:45.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:45.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:45 smithi149 ceph-mon[23948]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:05:45.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:05:45.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:05:45.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:46.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:46 smithi149 ceph-mon[23948]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:46.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:46 smithi069 ceph-mon[21859]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:49.354 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:49 smithi149 ceph-mon[23948]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:49.354 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:49 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:49.354 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:49 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:05:49.354 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:49 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:49.354 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:49 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:05:49.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:49 smithi069 ceph-mon[21859]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:49.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:49 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:49.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:49 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:05:49.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:49 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:49.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:49 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:05:50.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:50 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2909495092' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d98e167-3473-493e-8deb-1e10412bf5ff"}]: dispatch 2024-04-22T23:05:50.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:50 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2909495092' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7d98e167-3473-493e-8deb-1e10412bf5ff"}]': finished 2024-04-22T23:05:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:50 smithi149 ceph-mon[23948]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T23:05:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:50 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:05:50.333 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:50 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2909495092' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d98e167-3473-493e-8deb-1e10412bf5ff"}]: dispatch 2024-04-22T23:05:50.334 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:50 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2909495092' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7d98e167-3473-493e-8deb-1e10412bf5ff"}]': finished 2024-04-22T23:05:50.334 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:50 smithi069 ceph-mon[21859]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T23:05:50.334 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:50 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:05:51.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:51 smithi069 ceph-mon[21859]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:51.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:51 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:51.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:51 smithi069 ceph-mon[21859]: Updating smithi149:/etc/ceph/ceph.conf 2024-04-22T23:05:51.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:51 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/999926765' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:05:51.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:51 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:51.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:51 smithi149 ceph-mon[23948]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:51 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:51 smithi149 ceph-mon[23948]: Updating smithi149:/etc/ceph/ceph.conf 2024-04-22T23:05:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:51 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/999926765' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:05:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:51 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:52.584 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:52 smithi069 ceph-mon[21859]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:05:52.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:52 smithi149 ceph-mon[23948]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:05:53.522 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:53 smithi069 ceph-mon[21859]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:53.522 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:53 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:53.523 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:53 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:53.523 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:53 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:53.523 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:53 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:53.523 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:53 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:05:53.523 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:53 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T23:05:53.523 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:53 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:53.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:53 smithi149 ceph-mon[23948]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:53.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:53 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:53.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:53 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:53.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:53 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:53.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:53 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:53.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:53 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:05:53.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:53 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T23:05:53.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:53 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:54.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:54 smithi149 ceph-mon[23948]: Updating smithi069:/etc/ceph/ceph.conf 2024-04-22T23:05:54.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:54 smithi149 ceph-mon[23948]: Reconfiguring mon.smithi069 (unknown last config time)... 2024-04-22T23:05:54.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:54 smithi149 ceph-mon[23948]: Reconfiguring daemon mon.smithi069 on smithi069 2024-04-22T23:05:54.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:54 smithi069 ceph-mon[21859]: Updating smithi069:/etc/ceph/ceph.conf 2024-04-22T23:05:54.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:54 smithi069 ceph-mon[21859]: Reconfiguring mon.smithi069 (unknown last config time)... 2024-04-22T23:05:54.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:54 smithi069 ceph-mon[21859]: Reconfiguring daemon mon.smithi069 on smithi069 2024-04-22T23:05:55.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:55 smithi069 ceph-mon[21859]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:55.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:55 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:55.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:55 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.lyiiqg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:05:55.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:55 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:05:55.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:55 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:55.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:55 smithi149 ceph-mon[23948]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:55.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:55 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:55.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:55 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.lyiiqg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:05:55.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:55 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:05:55.618 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:55 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:56.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:56 smithi069 ceph-mon[21859]: Reconfiguring mgr.smithi069.lyiiqg (unknown last config time)... 2024-04-22T23:05:56.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:56 smithi069 ceph-mon[21859]: Reconfiguring daemon mgr.smithi069.lyiiqg on smithi069 2024-04-22T23:05:56.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:56 smithi149 ceph-mon[23948]: Reconfiguring mgr.smithi069.lyiiqg (unknown last config time)... 2024-04-22T23:05:56.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:56 smithi149 ceph-mon[23948]: Reconfiguring daemon mgr.smithi069.lyiiqg on smithi069 2024-04-22T23:05:57.313 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:57 smithi069 ceph-mon[21859]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:57.313 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:57.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:57 smithi149 ceph-mon[23948]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:57.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:05:58.372 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:58 smithi069 ceph-mon[21859]: Reconfiguring alertmanager.smithi069 (dependencies changed)... 2024-04-22T23:05:58.373 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:58 smithi069 ceph-mon[21859]: Reconfiguring daemon alertmanager.smithi069 on smithi069 2024-04-22T23:05:58.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:58 smithi149 ceph-mon[23948]: Reconfiguring alertmanager.smithi069 (dependencies changed)... 2024-04-22T23:05:58.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:58 smithi149 ceph-mon[23948]: Reconfiguring daemon alertmanager.smithi069 on smithi069 2024-04-22T23:05:59.537 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:59 smithi069 ceph-mon[21859]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:59.537 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:59 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T23:05:59.537 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:05:59 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:05:59.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:59 smithi149 ceph-mon[23948]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:05:59.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:59 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T23:05:59.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:05:59 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:00.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:00 smithi069 ceph-mon[21859]: Deploying daemon osd.0 on smithi069 2024-04-22T23:06:00.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:00 smithi149 ceph-mon[23948]: Deploying daemon osd.0 on smithi069 2024-04-22T23:06:01.566 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:01 smithi069 ceph-mon[21859]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:01.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:01 smithi149 ceph-mon[23948]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:03.628 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:03 smithi069 ceph-mon[21859]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:03.628 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:03 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:03.628 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:03 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:06:03.628 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:03 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:03.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:03 smithi149 ceph-mon[23948]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:03.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:03 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:03.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:03 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:06:03.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:03 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:04.505 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:04 smithi069 ceph-mon[21859]: Reconfiguring crash.smithi069 (monmap changed)... 2024-04-22T23:06:04.505 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:04 smithi069 ceph-mon[21859]: Reconfiguring daemon crash.smithi069 on smithi069 2024-04-22T23:06:04.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:04 smithi149 ceph-mon[23948]: Reconfiguring crash.smithi069 (monmap changed)... 2024-04-22T23:06:04.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:04 smithi149 ceph-mon[23948]: Reconfiguring daemon crash.smithi069 on smithi069 2024-04-22T23:06:05.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:05 smithi069 ceph-mon[21859]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:05.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:05 smithi149 ceph-mon[23948]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:07.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:07 smithi069 ceph-mon[21859]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:07.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:07 smithi149 ceph-mon[23948]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:09.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:09.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:08 smithi149 ceph-mon[23948]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:09.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:06:09.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:06:09.277 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:09.277 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:08 smithi069 ceph-mon[21859]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:09.278 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:06:09.278 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:06:10.133 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:10 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:10.133 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:10 smithi069 ceph-mon[21859]: Reconfiguring grafana.smithi069 (dependencies changed)... 2024-04-22T23:06:10.133 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:10 smithi069 ceph-mon[21859]: Reconfiguring daemon grafana.smithi069 on smithi069 2024-04-22T23:06:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:10 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:10 smithi149 ceph-mon[23948]: Reconfiguring grafana.smithi069 (dependencies changed)... 2024-04-22T23:06:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:10 smithi149 ceph-mon[23948]: Reconfiguring daemon grafana.smithi069 on smithi069 2024-04-22T23:06:11.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:11 smithi069 ceph-mon[21859]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:11.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:11 smithi069 ceph-mon[21859]: from='osd.0 [v2:172.21.15.69:6802/2446576616,v1:172.21.15.69:6803/2446576616]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T23:06:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:11 smithi149 ceph-mon[23948]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:11 smithi149 ceph-mon[23948]: from='osd.0 [v2:172.21.15.69:6802/2446576616,v1:172.21.15.69:6803/2446576616]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T23:06:12.390 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:12 smithi069 ceph-mon[21859]: from='osd.0 [v2:172.21.15.69:6802/2446576616,v1:172.21.15.69:6803/2446576616]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T23:06:12.390 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:12 smithi069 ceph-mon[21859]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T23:06:12.390 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:12 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:06:12.391 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:12 smithi069 ceph-mon[21859]: from='osd.0 [v2:172.21.15.69:6802/2446576616,v1:172.21.15.69:6803/2446576616]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:06:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:12 smithi149 ceph-mon[23948]: from='osd.0 [v2:172.21.15.69:6802/2446576616,v1:172.21.15.69:6803/2446576616]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T23:06:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:12 smithi149 ceph-mon[23948]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T23:06:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:12 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:06:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:12 smithi149 ceph-mon[23948]: from='osd.0 [v2:172.21.15.69:6802/2446576616,v1:172.21.15.69:6803/2446576616]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:06:13.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:13 smithi069 ceph-mon[21859]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:13.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:13 smithi069 ceph-mon[21859]: from='osd.0 [v2:172.21.15.69:6802/2446576616,v1:172.21.15.69:6803/2446576616]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-04-22T23:06:13.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:13 smithi069 ceph-mon[21859]: osdmap e8: 1 total, 0 up, 1 in 2024-04-22T23:06:13.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:13 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:06:13.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:13 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:06:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:13 smithi149 ceph-mon[23948]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:13 smithi149 ceph-mon[23948]: from='osd.0 [v2:172.21.15.69:6802/2446576616,v1:172.21.15.69:6803/2446576616]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-04-22T23:06:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:13 smithi149 ceph-mon[23948]: osdmap e8: 1 total, 0 up, 1 in 2024-04-22T23:06:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:13 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:06:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:13 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:06:14.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:14 smithi069 ceph-mon[21859]: purged_snaps scrub starts 2024-04-22T23:06:14.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:14 smithi069 ceph-mon[21859]: purged_snaps scrub ok 2024-04-22T23:06:14.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:06:14.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:14 smithi069 ceph-mon[21859]: osd.0 [v2:172.21.15.69:6802/2446576616,v1:172.21.15.69:6803/2446576616] boot 2024-04-22T23:06:14.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:14 smithi069 ceph-mon[21859]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T23:06:14.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:06:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:14 smithi149 ceph-mon[23948]: purged_snaps scrub starts 2024-04-22T23:06:14.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:14 smithi149 ceph-mon[23948]: purged_snaps scrub ok 2024-04-22T23:06:14.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:14 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:06:14.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:14 smithi149 ceph-mon[23948]: osd.0 [v2:172.21.15.69:6802/2446576616,v1:172.21.15.69:6803/2446576616] boot 2024-04-22T23:06:14.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:14 smithi149 ceph-mon[23948]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T23:06:14.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:14 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:06:15.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:15 smithi069 ceph-mon[21859]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:15.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:15 smithi069 ceph-mon[21859]: osdmap e10: 1 total, 1 up, 1 in 2024-04-22T23:06:15.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:15 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:15.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:15 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:06:15.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:15 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:15.352 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:15 smithi149 ceph-mon[23948]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T23:06:15.352 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:15 smithi149 ceph-mon[23948]: osdmap e10: 1 total, 1 up, 1 in 2024-04-22T23:06:15.352 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:15 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:15.352 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:15 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:06:15.352 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:15 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:16.277 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:16 smithi149 ceph-mon[23948]: Reconfiguring crash.smithi149 (monmap changed)... 2024-04-22T23:06:16.277 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:16 smithi149 ceph-mon[23948]: Reconfiguring daemon crash.smithi149 on smithi149 2024-04-22T23:06:16.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:16 smithi069 ceph-mon[21859]: Reconfiguring crash.smithi149 (monmap changed)... 2024-04-22T23:06:16.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:16 smithi069 ceph-mon[21859]: Reconfiguring daemon crash.smithi149 on smithi149 2024-04-22T23:06:16.479 INFO:teuthology.orchestra.run.smithi069.stdout:Created osd(s) 0 on host 'smithi069' 2024-04-22T23:06:17.084 DEBUG:teuthology.orchestra.run.smithi069:osd.0> sudo journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.0.service 2024-04-22T23:06:17.086 INFO:tasks.cephadm:Deploying osd.1 on smithi069 with /dev/vg_nvme/lv_3... 2024-04-22T23:06:17.087 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-22T23:06:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:17 smithi149 ceph-mon[23948]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:17 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:17 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.rxqkqk", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:06:17.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:17 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:06:17.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:17 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:17.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:17 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:17.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:17 smithi069 ceph-mon[21859]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:17.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:17 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:17.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:17 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.rxqkqk", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:06:17.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:17 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:06:17.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:17 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:17.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:17 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:18.178 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:18 smithi149 ceph-mon[23948]: Reconfiguring mgr.smithi149.rxqkqk (monmap changed)... 2024-04-22T23:06:18.178 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:18 smithi149 ceph-mon[23948]: Reconfiguring daemon mgr.smithi149.rxqkqk on smithi149 2024-04-22T23:06:18.178 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:18 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:18.179 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:18 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:06:18.179 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:18 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T23:06:18.179 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:18 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:18.356 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:18 smithi069 ceph-mon[21859]: Reconfiguring mgr.smithi149.rxqkqk (monmap changed)... 2024-04-22T23:06:18.356 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:18 smithi069 ceph-mon[21859]: Reconfiguring daemon mgr.smithi149.rxqkqk on smithi149 2024-04-22T23:06:18.356 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:18 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:18.356 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:18 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:06:18.356 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:18 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T23:06:18.356 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:18 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:19.153 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-22T23:06:19.153 INFO:teuthology.orchestra.run.smithi069.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-22T23:06:19.153 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10+0 records in 2024-04-22T23:06:19.153 INFO:teuthology.orchestra.run.smithi069.stderr:10+0 records out 2024-04-22T23:06:19.153 INFO:teuthology.orchestra.run.smithi069.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0134141 s, 782 MB/s 2024-04-22T23:06:19.153 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping successful for: 2024-04-22T23:06:19.179 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: Reconfiguring mon.smithi149 (monmap changed)... 2024-04-22T23:06:19.179 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: Reconfiguring daemon mon.smithi149 on smithi149 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.69:9093"}]: dispatch 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.69:3000"}]: dispatch 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.69:9095"}]: dispatch 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:19.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:06:19.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: Reconfiguring mon.smithi149 (monmap changed)... 2024-04-22T23:06:19.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: Reconfiguring daemon mon.smithi149 on smithi149 2024-04-22T23:06:19.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:19.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:19.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T23:06:19.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.69:9093"}]: dispatch 2024-04-22T23:06:19.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:19.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T23:06:19.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.69:3000"}]: dispatch 2024-04-22T23:06:19.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:19.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:06:19.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.69:9095"}]: dispatch 2024-04-22T23:06:19.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:19.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:06:19.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:19.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:06:19.793 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi069:vg_nvme/lv_3 2024-04-22T23:06:20.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:20 smithi069 ceph-mon[21859]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T23:06:20.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:20 smithi069 ceph-mon[21859]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.69:9093"}]: dispatch 2024-04-22T23:06:20.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:20 smithi069 ceph-mon[21859]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T23:06:20.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:20 smithi069 ceph-mon[21859]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.69:3000"}]: dispatch 2024-04-22T23:06:20.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:20 smithi069 ceph-mon[21859]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:06:20.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:20 smithi069 ceph-mon[21859]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.69:9095"}]: dispatch 2024-04-22T23:06:20.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:20 smithi149 ceph-mon[23948]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T23:06:20.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:20 smithi149 ceph-mon[23948]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.69:9093"}]: dispatch 2024-04-22T23:06:20.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:20 smithi149 ceph-mon[23948]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T23:06:20.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:20 smithi149 ceph-mon[23948]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.69:3000"}]: dispatch 2024-04-22T23:06:20.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:20 smithi149 ceph-mon[23948]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:06:20.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:20 smithi149 ceph-mon[23948]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.69:9095"}]: dispatch 2024-04-22T23:06:21.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:21 smithi149 ceph-mon[23948]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:21.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:21 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:21.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:21 smithi069 ceph-mon[21859]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:21.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:21 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:23.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:23 smithi069 ceph-mon[21859]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:23.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:23 smithi149 ceph-mon[23948]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:24.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:24 smithi069 ceph-mon[21859]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:06:24.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:24 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:06:24.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:24 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:06:24.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:24 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:24.352 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:24 smithi149 ceph-mon[23948]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:06:24.352 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:24 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:06:24.352 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:24 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:06:24.352 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:24 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:25.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:25 smithi069 ceph-mon[21859]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:25.363 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:25 smithi149 ceph-mon[23948]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:27.308 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:27 smithi069 ceph-mon[21859]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:27.308 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:27 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:27.308 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:27 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2245880178' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d15522c-2ca4-451c-a569-f31e25e5a08b"}]: dispatch 2024-04-22T23:06:27.308 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:27 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2245880178' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9d15522c-2ca4-451c-a569-f31e25e5a08b"}]': finished 2024-04-22T23:06:27.308 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:27 smithi069 ceph-mon[21859]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T23:06:27.308 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:27 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:06:27.308 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:27 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3576281468' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:06:27.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:27 smithi149 ceph-mon[23948]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:27.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:27 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:27.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:27 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2245880178' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d15522c-2ca4-451c-a569-f31e25e5a08b"}]: dispatch 2024-04-22T23:06:27.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:27 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2245880178' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9d15522c-2ca4-451c-a569-f31e25e5a08b"}]': finished 2024-04-22T23:06:27.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:27 smithi149 ceph-mon[23948]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T23:06:27.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:27 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:06:27.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:27 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/3576281468' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:06:29.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:29 smithi069 ceph-mon[21859]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:29 smithi149 ceph-mon[23948]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:31.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:31 smithi069 ceph-mon[21859]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:31 smithi149 ceph-mon[23948]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:32.967 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:32 smithi149 ceph-mon[23948]: Detected new or changed devices on smithi069 2024-04-22T23:06:32.967 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:32 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:32.967 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:32 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:06:32.967 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:32 smithi149 ceph-mon[23948]: Adjusting osd_memory_target on smithi069 to 12864M 2024-04-22T23:06:32.967 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:32 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:32.968 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:32 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:32.968 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:32 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:06:32.968 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:32 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:32.968 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:32 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:06:32.968 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:32 smithi149 ceph-mon[23948]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:33.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:32 smithi069 ceph-mon[21859]: Detected new or changed devices on smithi069 2024-04-22T23:06:33.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:32 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:33.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:32 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:06:33.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:32 smithi069 ceph-mon[21859]: Adjusting osd_memory_target on smithi069 to 12864M 2024-04-22T23:06:33.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:32 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:33.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:32 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:33.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:32 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:06:33.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:32 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:33.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:32 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:06:33.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:32 smithi069 ceph-mon[21859]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:34.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:34 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:34.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:34 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T23:06:34.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:34 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:34.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:34 smithi149 ceph-mon[23948]: Deploying daemon osd.1 on smithi069 2024-04-22T23:06:34.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:34 smithi149 ceph-mon[23948]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:34.725 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:34 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:34.725 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:34 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T23:06:34.725 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:34 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:34.725 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:34 smithi069 ceph-mon[21859]: Deploying daemon osd.1 on smithi069 2024-04-22T23:06:34.725 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:34 smithi069 ceph-mon[21859]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:37.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:37 smithi069 ceph-mon[21859]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:37 smithi149 ceph-mon[23948]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:39.298 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:39 smithi069 ceph-mon[21859]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:39.298 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:39 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:39.298 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:39 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:39 smithi149 ceph-mon[23948]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:39.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:39 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:39.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:39 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:41.339 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:41 smithi069 ceph-mon[21859]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:41.339 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:41 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:41.339 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:41 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:06:41.339 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:41 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:41.339 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:41 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:06:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:41 smithi149 ceph-mon[23948]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:41 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:41.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:41 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:06:41.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:41 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:41.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:41 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:06:43.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:43 smithi069 ceph-mon[21859]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:43.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:43 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:43 smithi149 ceph-mon[23948]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:43 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:43.972 INFO:teuthology.orchestra.run.smithi069.stdout:Created osd(s) 1 on host 'smithi069' 2024-04-22T23:06:44.664 DEBUG:teuthology.orchestra.run.smithi069:osd.1> sudo journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.1.service 2024-04-22T23:06:44.666 INFO:tasks.cephadm:Deploying osd.2 on smithi069 with /dev/vg_nvme/lv_2... 2024-04-22T23:06:44.666 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-22T23:06:45.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:44 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:45.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:44 smithi069 ceph-mon[21859]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:45.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:44 smithi069 ceph-mon[21859]: from='osd.1 [v2:172.21.15.69:6810/2958036688,v1:172.21.15.69:6811/2958036688]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T23:06:45.269 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:44 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:45.269 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:44 smithi149 ceph-mon[23948]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:45.269 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:44 smithi149 ceph-mon[23948]: from='osd.1 [v2:172.21.15.69:6810/2958036688,v1:172.21.15.69:6811/2958036688]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T23:06:46.355 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:45 smithi069 ceph-mon[21859]: from='osd.1 [v2:172.21.15.69:6810/2958036688,v1:172.21.15.69:6811/2958036688]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T23:06:46.355 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:45 smithi069 ceph-mon[21859]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T23:06:46.355 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:06:46.355 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:45 smithi069 ceph-mon[21859]: from='osd.1 [v2:172.21.15.69:6810/2958036688,v1:172.21.15.69:6811/2958036688]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:06:46.355 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:06:45 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[36372]: 2024-04-22T23:06:45.979+0000 7f1d71fca700 -1 osd.1 0 waiting for initial osdmap 2024-04-22T23:06:46.355 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:06:45 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[36372]: 2024-04-22T23:06:45.982+0000 7f1d6c961700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:06:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:45 smithi149 ceph-mon[23948]: from='osd.1 [v2:172.21.15.69:6810/2958036688,v1:172.21.15.69:6811/2958036688]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T23:06:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:45 smithi149 ceph-mon[23948]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T23:06:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:06:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:45 smithi149 ceph-mon[23948]: from='osd.1 [v2:172.21.15.69:6810/2958036688,v1:172.21.15.69:6811/2958036688]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:06:47.066 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-22T23:06:47.066 INFO:teuthology.orchestra.run.smithi069.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-22T23:06:47.066 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10+0 records in 2024-04-22T23:06:47.066 INFO:teuthology.orchestra.run.smithi069.stderr:10+0 records out 2024-04-22T23:06:47.066 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122528 s, 856 MB/s 2024-04-22T23:06:47.066 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping successful for: 2024-04-22T23:06:47.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:46 smithi069 ceph-mon[21859]: from='osd.1 [v2:172.21.15.69:6810/2958036688,v1:172.21.15.69:6811/2958036688]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-04-22T23:06:47.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:46 smithi069 ceph-mon[21859]: osdmap e13: 2 total, 1 up, 2 in 2024-04-22T23:06:47.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:46 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:06:47.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:46 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:06:47.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:46 smithi069 ceph-mon[21859]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:47.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:46 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:06:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:46 smithi149 ceph-mon[23948]: from='osd.1 [v2:172.21.15.69:6810/2958036688,v1:172.21.15.69:6811/2958036688]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-04-22T23:06:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:46 smithi149 ceph-mon[23948]: osdmap e13: 2 total, 1 up, 2 in 2024-04-22T23:06:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:46 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:06:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:46 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:06:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:46 smithi149 ceph-mon[23948]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-22T23:06:47.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:46 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:06:47.875 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi069:vg_nvme/lv_2 2024-04-22T23:06:48.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:47 smithi069 ceph-mon[21859]: purged_snaps scrub starts 2024-04-22T23:06:48.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:47 smithi069 ceph-mon[21859]: purged_snaps scrub ok 2024-04-22T23:06:48.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:47 smithi069 ceph-mon[21859]: osd.1 [v2:172.21.15.69:6810/2958036688,v1:172.21.15.69:6811/2958036688] boot 2024-04-22T23:06:48.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:47 smithi069 ceph-mon[21859]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T23:06:48.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:47 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:06:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:47 smithi149 ceph-mon[23948]: purged_snaps scrub starts 2024-04-22T23:06:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:47 smithi149 ceph-mon[23948]: purged_snaps scrub ok 2024-04-22T23:06:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:47 smithi149 ceph-mon[23948]: osd.1 [v2:172.21.15.69:6810/2958036688,v1:172.21.15.69:6811/2958036688] boot 2024-04-22T23:06:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:47 smithi149 ceph-mon[23948]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T23:06:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:47 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:06:49.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:48 smithi069 ceph-mon[21859]: osdmap e15: 2 total, 2 up, 2 in 2024-04-22T23:06:49.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:48 smithi069 ceph-mon[21859]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:06:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:48 smithi149 ceph-mon[23948]: osdmap e15: 2 total, 2 up, 2 in 2024-04-22T23:06:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:48 smithi149 ceph-mon[23948]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:06:51.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:51 smithi069 ceph-mon[21859]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:06:51.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:51 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:06:51.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:51 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:06:51.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:51 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:51.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:51 smithi149 ceph-mon[23948]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:06:51.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:51 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:06:51.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:51 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:06:51.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:51 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:06:52.204 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:52 smithi069 ceph-mon[21859]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:06:52.204 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:52 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:52.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:52 smithi149 ceph-mon[23948]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:06:52.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:52 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:06:53.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:53 smithi069 ceph-mon[21859]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:06:53.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:53 smithi149 ceph-mon[23948]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:06:54.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:54 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1485143737' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14d41cfc-5bca-40a5-bede-a02925fa4cb7"}]: dispatch 2024-04-22T23:06:54.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:54 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1485143737' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "14d41cfc-5bca-40a5-bede-a02925fa4cb7"}]': finished 2024-04-22T23:06:54.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:54 smithi069 ceph-mon[21859]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T23:06:54.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:06:54.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:54 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3322207720' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:06:54.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:54 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/1485143737' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14d41cfc-5bca-40a5-bede-a02925fa4cb7"}]: dispatch 2024-04-22T23:06:54.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:54 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/1485143737' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "14d41cfc-5bca-40a5-bede-a02925fa4cb7"}]': finished 2024-04-22T23:06:54.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:54 smithi149 ceph-mon[23948]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T23:06:54.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:06:54.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:54 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/3322207720' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:06:55.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:55 smithi149 ceph-mon[23948]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:06:55.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:55 smithi069 ceph-mon[21859]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:06:57.431 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:57 smithi069 ceph-mon[21859]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:06:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:57 smithi149 ceph-mon[23948]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:06:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:06:59 smithi149 ceph-mon[23948]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:06:59.532 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:06:59 smithi069 ceph-mon[21859]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:00.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:00 smithi069 ceph-mon[21859]: Detected new or changed devices on smithi069 2024-04-22T23:07:00.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:00 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:00.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:00 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:00.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:00 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:00.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:00 smithi069 ceph-mon[21859]: Adjusting osd_memory_target on smithi069 to 6432M 2024-04-22T23:07:00.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:00 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:00.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:00 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:00.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:00 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:07:00.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:00 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:00.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:00 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:07:00.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:00 smithi149 ceph-mon[23948]: Detected new or changed devices on smithi069 2024-04-22T23:07:00.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:00 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:00.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:00 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:00.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:00 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:00.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:00 smithi149 ceph-mon[23948]: Adjusting osd_memory_target on smithi069 to 6432M 2024-04-22T23:07:00.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:00 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:00.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:00 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:00.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:00 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:07:00.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:00 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:00.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:00 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:07:01.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:01 smithi069 ceph-mon[21859]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:01.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:01 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T23:07:01.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:01 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:01.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:01 smithi149 ceph-mon[23948]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:01.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:01 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T23:07:01.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:01 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:02.356 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:02 smithi069 ceph-mon[21859]: Deploying daemon osd.2 on smithi069 2024-04-22T23:07:02.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:02 smithi149 ceph-mon[23948]: Deploying daemon osd.2 on smithi069 2024-04-22T23:07:03.486 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:03 smithi069 ceph-mon[21859]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:03.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:03 smithi149 ceph-mon[23948]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:04.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:04 smithi149 ceph-mon[23948]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:04.727 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:04 smithi069 ceph-mon[21859]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:07.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:06 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:07.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:07 smithi069 ceph-mon[21859]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:07.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:07 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:07.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:07 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:07.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:07 smithi149 ceph-mon[23948]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:07.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:07 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:09.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:09.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:08 smithi069 ceph-mon[21859]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:09.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:07:09.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:07:09.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:07:09.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:09.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:07:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:08 smithi149 ceph-mon[23948]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:07:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:07:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:07:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:07:11.341 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:11 smithi069 ceph-mon[21859]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:11 smithi149 ceph-mon[23948]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:11.985 INFO:teuthology.orchestra.run.smithi069.stdout:Created osd(s) 2 on host 'smithi069' 2024-04-22T23:07:12.298 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:12 smithi069 ceph-mon[21859]: from='osd.2 [v2:172.21.15.69:6818/2823169129,v1:172.21.15.69:6819/2823169129]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T23:07:12.298 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:12 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:12 smithi149 ceph-mon[23948]: from='osd.2 [v2:172.21.15.69:6818/2823169129,v1:172.21.15.69:6819/2823169129]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T23:07:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:12 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:12.598 DEBUG:teuthology.orchestra.run.smithi069:osd.2> sudo journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.2.service 2024-04-22T23:07:12.601 INFO:tasks.cephadm:Deploying osd.3 on smithi069 with /dev/vg_nvme/lv_1... 2024-04-22T23:07:12.601 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-22T23:07:13.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:13 smithi069 ceph-mon[21859]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:13.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:13 smithi069 ceph-mon[21859]: from='osd.2 [v2:172.21.15.69:6818/2823169129,v1:172.21.15.69:6819/2823169129]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T23:07:13.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:13 smithi069 ceph-mon[21859]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T23:07:13.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:13 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:07:13.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:13 smithi069 ceph-mon[21859]: from='osd.2 [v2:172.21.15.69:6818/2823169129,v1:172.21.15.69:6819/2823169129]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:07:13.147 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:07:13 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2[40079]: 2024-04-22T23:07:13.071+0000 7fbbc44f2700 -1 osd.2 0 waiting for initial osdmap 2024-04-22T23:07:13.147 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:07:13 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2[40079]: 2024-04-22T23:07:13.077+0000 7fbbbe688700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:07:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:13 smithi149 ceph-mon[23948]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:13 smithi149 ceph-mon[23948]: from='osd.2 [v2:172.21.15.69:6818/2823169129,v1:172.21.15.69:6819/2823169129]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T23:07:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:13 smithi149 ceph-mon[23948]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T23:07:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:13 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:07:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:13 smithi149 ceph-mon[23948]: from='osd.2 [v2:172.21.15.69:6818/2823169129,v1:172.21.15.69:6819/2823169129]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:07:14.340 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:14 smithi069 ceph-mon[21859]: from='osd.2 [v2:172.21.15.69:6818/2823169129,v1:172.21.15.69:6819/2823169129]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-04-22T23:07:14.340 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:14 smithi069 ceph-mon[21859]: osdmap e18: 3 total, 2 up, 3 in 2024-04-22T23:07:14.340 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:07:14.341 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:07:14.341 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:07:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:14 smithi149 ceph-mon[23948]: from='osd.2 [v2:172.21.15.69:6818/2823169129,v1:172.21.15.69:6819/2823169129]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-04-22T23:07:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:14 smithi149 ceph-mon[23948]: osdmap e18: 3 total, 2 up, 3 in 2024-04-22T23:07:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:14 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:07:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:14 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:07:14.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:14 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:07:15.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:15 smithi069 ceph-mon[21859]: purged_snaps scrub starts 2024-04-22T23:07:15.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:15 smithi069 ceph-mon[21859]: purged_snaps scrub ok 2024-04-22T23:07:15.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:15 smithi069 ceph-mon[21859]: pgmap v69: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:15.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:15 smithi069 ceph-mon[21859]: osd.2 [v2:172.21.15.69:6818/2823169129,v1:172.21.15.69:6819/2823169129] boot 2024-04-22T23:07:15.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:15 smithi069 ceph-mon[21859]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T23:07:15.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:15 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:07:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:15 smithi149 ceph-mon[23948]: purged_snaps scrub starts 2024-04-22T23:07:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:15 smithi149 ceph-mon[23948]: purged_snaps scrub ok 2024-04-22T23:07:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:15 smithi149 ceph-mon[23948]: pgmap v69: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-04-22T23:07:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:15 smithi149 ceph-mon[23948]: osd.2 [v2:172.21.15.69:6818/2823169129,v1:172.21.15.69:6819/2823169129] boot 2024-04-22T23:07:15.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:15 smithi149 ceph-mon[23948]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T23:07:15.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:15 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:07:16.531 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-22T23:07:16.532 INFO:teuthology.orchestra.run.smithi069.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-22T23:07:16.532 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10+0 records in 2024-04-22T23:07:16.532 INFO:teuthology.orchestra.run.smithi069.stderr:10+0 records out 2024-04-22T23:07:16.532 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0148848 s, 704 MB/s 2024-04-22T23:07:16.532 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping successful for: 2024-04-22T23:07:16.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:16 smithi149 ceph-mon[23948]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T23:07:16.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:16 smithi149 ceph-mon[23948]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:16.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T23:07:16.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:16.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:16 smithi069 ceph-mon[21859]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T23:07:16.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:16 smithi069 ceph-mon[21859]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:16.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T23:07:16.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:17.195 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi069:vg_nvme/lv_1 2024-04-22T23:07:17.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:17 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T23:07:17.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:17 smithi149 ceph-mon[23948]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T23:07:17.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:17 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T23:07:17.775 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:17 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T23:07:17.775 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:17 smithi069 ceph-mon[21859]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T23:07:17.775 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:17 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T23:07:18.094 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:07:17 smithi069 sudo[41550]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T23:07:18.094 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:07:17 smithi069 sudo[41550]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T23:07:18.094 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:07:17 smithi069 sudo[41550]: pam_unix(sudo:session): session closed for user root 2024-04-22T23:07:18.094 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:07:18 smithi069 sudo[41568]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T23:07:18.361 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:07:18 smithi069 sudo[41568]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T23:07:18.361 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:07:18 smithi069 sudo[41568]: pam_unix(sudo:session): session closed for user root 2024-04-22T23:07:18.620 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:07:18 smithi069 sudo[41575]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T23:07:18.620 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:07:18 smithi069 sudo[41575]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T23:07:18.620 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:07:18 smithi069 sudo[41575]: pam_unix(sudo:session): session closed for user root 2024-04-22T23:07:18.620 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:18 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T23:07:18.620 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:18 smithi069 ceph-mon[21859]: osdmap e22: 3 total, 3 up, 3 in 2024-04-22T23:07:18.621 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:18 smithi069 ceph-mon[21859]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:18.621 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:18 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:18.870 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:07:18 smithi069 sudo[41591]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T23:07:18.871 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:07:18 smithi069 sudo[41591]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T23:07:18.871 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:07:18 smithi069 sudo[41591]: pam_unix(sudo:session): session closed for user root 2024-04-22T23:07:18.871 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:07:18 smithi069 sudo[41597]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T23:07:18.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:18 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T23:07:18.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:18 smithi149 ceph-mon[23948]: osdmap e22: 3 total, 3 up, 3 in 2024-04-22T23:07:18.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:18 smithi149 ceph-mon[23948]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:18.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:18 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:19.130 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:07:18 smithi069 sudo[41597]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T23:07:19.131 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:07:18 smithi069 sudo[41597]: pam_unix(sudo:session): session closed for user root 2024-04-22T23:07:19.392 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:07:19 smithi069 sudo[41620]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T23:07:19.393 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:07:19 smithi069 sudo[41620]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T23:07:19.393 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:07:19 smithi069 sudo[41620]: pam_unix(sudo:session): session closed for user root 2024-04-22T23:07:19.393 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:19 smithi069 sudo[41623]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T23:07:19.393 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:19 smithi069 sudo[41623]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T23:07:19.393 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:19 smithi069 sudo[41623]: pam_unix(sudo:session): session closed for user root 2024-04-22T23:07:19.642 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:19 smithi069 sudo[41646]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T23:07:19.643 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:19 smithi069 sudo[41646]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T23:07:19.643 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:19 smithi069 sudo[41646]: pam_unix(sudo:session): session closed for user root 2024-04-22T23:07:19.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:19 smithi069 ceph-mon[21859]: osdmap e23: 3 total, 3 up, 3 in 2024-04-22T23:07:19.898 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:19 smithi149 ceph-mon[23948]: osdmap e23: 3 total, 3 up, 3 in 2024-04-22T23:07:19.898 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:19 smithi149 sudo[26253]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T23:07:19.898 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:19 smithi149 sudo[26253]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T23:07:19.898 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:19 smithi149 sudo[26253]: pam_unix(sudo:session): session closed for user root 2024-04-22T23:07:20.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 sudo[26256]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T23:07:20.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 sudo[26256]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T23:07:20.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 sudo[26256]: pam_unix(sudo:session): session closed for user root 2024-04-22T23:07:20.881 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T23:07:20.881 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T23:07:20.881 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:07:20.882 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:07:20.882 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T23:07:20.882 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:07:20.882 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:07:20.882 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: pgmap v77: 1 pgs: 1 active+clean; 449 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:20.883 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T23:07:20.883 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: mgrmap e19: smithi069.lyiiqg(active, since 2m), standbys: smithi149.rxqkqk 2024-04-22T23:07:20.883 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:20.883 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:07:20.883 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:07:20.883 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:20 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:20.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T23:07:20.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T23:07:20.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:07:20.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:07:20.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T23:07:20.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:07:20.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:07:20.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: pgmap v77: 1 pgs: 1 active+clean; 449 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:20.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T23:07:20.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: mgrmap e19: smithi069.lyiiqg(active, since 2m), standbys: smithi149.rxqkqk 2024-04-22T23:07:20.953 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:20.953 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:07:20.953 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:07:20.953 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:20 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:22.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:22 smithi069 ceph-mon[21859]: from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:07:22.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:22.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:22.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:22.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:22.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:22 smithi069 ceph-mon[21859]: Adjusting osd_memory_target on smithi069 to 4288M 2024-04-22T23:07:22.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:22.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:22.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:07:22.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:22.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:07:22.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:22 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:22.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:22 smithi149 ceph-mon[23948]: from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:07:22.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:22 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:22.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:22 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:22.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:22 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:22.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:22 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:22.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:22 smithi149 ceph-mon[23948]: Adjusting osd_memory_target on smithi069 to 4288M 2024-04-22T23:07:22.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:22 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:22.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:22 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:22.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:22 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:07:22.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:22 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:22.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:22 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:07:22.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:22 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:23.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:23 smithi069 ceph-mon[21859]: pgmap v78: 1 pgs: 1 active+clean; 449 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:23 smithi149 ceph-mon[23948]: pgmap v78: 1 pgs: 1 active+clean; 449 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:24.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:24 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2079588481' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40b632b4-3632-4281-bde9-e7aeadecf33f"}]: dispatch 2024-04-22T23:07:24.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:24 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2079588481' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40b632b4-3632-4281-bde9-e7aeadecf33f"}]': finished 2024-04-22T23:07:24.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:24 smithi149 ceph-mon[23948]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T23:07:24.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:24 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:07:24.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:24 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/1567629131' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:07:24.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:24 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2079588481' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40b632b4-3632-4281-bde9-e7aeadecf33f"}]: dispatch 2024-04-22T23:07:24.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:24 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2079588481' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40b632b4-3632-4281-bde9-e7aeadecf33f"}]': finished 2024-04-22T23:07:24.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:24 smithi069 ceph-mon[21859]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T23:07:24.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:24 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:07:24.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:24 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1567629131' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:07:25.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:25 smithi069 ceph-mon[21859]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2024-04-22T23:07:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:25 smithi149 ceph-mon[23948]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2024-04-22T23:07:27.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:27 smithi069 ceph-mon[21859]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2024-04-22T23:07:27.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:27 smithi149 ceph-mon[23948]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2024-04-22T23:07:29.446 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:29 smithi069 ceph-mon[21859]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.8 KiB/s rd, 137 KiB/s wr, 9 op/s 2024-04-22T23:07:29.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:29 smithi149 ceph-mon[23948]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.8 KiB/s rd, 137 KiB/s wr, 9 op/s 2024-04-22T23:07:30.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:30 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T23:07:30.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:30 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:30.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:30 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T23:07:30.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:30 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:31.508 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:31 smithi069 ceph-mon[21859]: Deploying daemon osd.3 on smithi069 2024-04-22T23:07:31.508 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:31 smithi069 ceph-mon[21859]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-04-22T23:07:31.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:31 smithi149 ceph-mon[23948]: Deploying daemon osd.3 on smithi069 2024-04-22T23:07:31.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:31 smithi149 ceph-mon[23948]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-04-22T23:07:33.484 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:33 smithi069 ceph-mon[21859]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-04-22T23:07:33.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:33 smithi149 ceph-mon[23948]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-04-22T23:07:34.693 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:34 smithi069 ceph-mon[21859]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:34.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:34 smithi149 ceph-mon[23948]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:35.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:35 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:35.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:35 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:07:35.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:35 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:35.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:35 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:07:35.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:35 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:35.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:35 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:07:35.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:35 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:35.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:35 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:07:36.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:36 smithi149 ceph-mon[23948]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:37.077 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:36 smithi069 ceph-mon[21859]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:37.400 INFO:teuthology.orchestra.run.smithi069.stdout:Created osd(s) 3 on host 'smithi069' 2024-04-22T23:07:38.197 DEBUG:teuthology.orchestra.run.smithi069:osd.3> sudo journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.3.service 2024-04-22T23:07:38.200 INFO:tasks.cephadm:Deploying osd.4 on smithi149 with /dev/vg_nvme/lv_4... 2024-04-22T23:07:38.200 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-22T23:07:38.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:38 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:38.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:38 smithi149 ceph-mon[23948]: from='osd.3 [v2:172.21.15.69:6826/1031635705,v1:172.21.15.69:6827/1031635705]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T23:07:38.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:38 smithi149 ceph-mon[23948]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:38.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:38 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:38.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:38 smithi069 ceph-mon[21859]: from='osd.3 [v2:172.21.15.69:6826/1031635705,v1:172.21.15.69:6827/1031635705]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T23:07:38.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:38 smithi069 ceph-mon[21859]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:39.677 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:39 smithi149 ceph-mon[23948]: from='osd.3 [v2:172.21.15.69:6826/1031635705,v1:172.21.15.69:6827/1031635705]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T23:07:39.678 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:39 smithi149 ceph-mon[23948]: osdmap e25: 4 total, 3 up, 4 in 2024-04-22T23:07:39.678 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:39 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:07:39.678 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:39 smithi149 ceph-mon[23948]: from='osd.3 [v2:172.21.15.69:6826/1031635705,v1:172.21.15.69:6827/1031635705]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:07:39.756 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:39 smithi069 ceph-mon[21859]: from='osd.3 [v2:172.21.15.69:6826/1031635705,v1:172.21.15.69:6827/1031635705]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T23:07:39.757 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:39 smithi069 ceph-mon[21859]: osdmap e25: 4 total, 3 up, 4 in 2024-04-22T23:07:39.757 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:39 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:07:39.757 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:39 smithi069 ceph-mon[21859]: from='osd.3 [v2:172.21.15.69:6826/1031635705,v1:172.21.15.69:6827/1031635705]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:07:39.757 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:07:39 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3[43345]: 2024-04-22T23:07:39.410+0000 7f5fea07b700 -1 osd.3 0 waiting for initial osdmap 2024-04-22T23:07:39.757 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:07:39 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3[43345]: 2024-04-22T23:07:39.414+0000 7f5fe3a10700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:07:40.159 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-22T23:07:40.159 INFO:teuthology.orchestra.run.smithi149.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-22T23:07:40.159 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 10+0 records in 2024-04-22T23:07:40.159 INFO:teuthology.orchestra.run.smithi149.stderr:10+0 records out 2024-04-22T23:07:40.159 INFO:teuthology.orchestra.run.smithi149.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0129996 s, 807 MB/s 2024-04-22T23:07:40.159 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 2024-04-22T23:07:40.160 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping successful for: 2024-04-22T23:07:40.534 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:40 smithi149 ceph-mon[23948]: from='osd.3 [v2:172.21.15.69:6826/1031635705,v1:172.21.15.69:6827/1031635705]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-04-22T23:07:40.535 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:40 smithi149 ceph-mon[23948]: osdmap e26: 4 total, 3 up, 4 in 2024-04-22T23:07:40.535 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:40 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:07:40.535 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:40 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:07:40.535 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:40 smithi149 ceph-mon[23948]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:40.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:40 smithi069 ceph-mon[21859]: from='osd.3 [v2:172.21.15.69:6826/1031635705,v1:172.21.15.69:6827/1031635705]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-04-22T23:07:40.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:40 smithi069 ceph-mon[21859]: osdmap e26: 4 total, 3 up, 4 in 2024-04-22T23:07:40.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:40 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:07:40.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:40 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:07:40.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:40 smithi069 ceph-mon[21859]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T23:07:40.784 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi149:vg_nvme/lv_4 2024-04-22T23:07:41.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:41 smithi149 ceph-mon[23948]: purged_snaps scrub starts 2024-04-22T23:07:41.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:41 smithi149 ceph-mon[23948]: purged_snaps scrub ok 2024-04-22T23:07:41.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:41 smithi149 ceph-mon[23948]: osd.3 [v2:172.21.15.69:6826/1031635705,v1:172.21.15.69:6827/1031635705] boot 2024-04-22T23:07:41.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:41 smithi149 ceph-mon[23948]: osdmap e27: 4 total, 4 up, 4 in 2024-04-22T23:07:41.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:41 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:07:41.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:41 smithi069 ceph-mon[21859]: purged_snaps scrub starts 2024-04-22T23:07:41.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:41 smithi069 ceph-mon[21859]: purged_snaps scrub ok 2024-04-22T23:07:41.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:41 smithi069 ceph-mon[21859]: osd.3 [v2:172.21.15.69:6826/1031635705,v1:172.21.15.69:6827/1031635705] boot 2024-04-22T23:07:41.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:41 smithi069 ceph-mon[21859]: osdmap e27: 4 total, 4 up, 4 in 2024-04-22T23:07:41.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:41 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:07:42.510 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:42 smithi069 ceph-mon[21859]: osdmap e28: 4 total, 4 up, 4 in 2024-04-22T23:07:42.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:42 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:42.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:42 smithi069 ceph-mon[21859]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:42.642 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:42 smithi149 ceph-mon[23948]: osdmap e28: 4 total, 4 up, 4 in 2024-04-22T23:07:42.642 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:42 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:42.642 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:42 smithi149 ceph-mon[23948]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:43.891 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:43 smithi149 ceph-mon[23948]: from='client.24137 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:07:43.891 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:43 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:07:43.891 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:43 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:07:43.892 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:43 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:43.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:43 smithi069 ceph-mon[21859]: from='client.24137 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:07:43.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:43 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:07:43.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:43 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:07:43.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:43 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:44.865 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:44 smithi069 ceph-mon[21859]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:44.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:44 smithi149 ceph-mon[23948]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:46.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: Detected new or changed devices on smithi069 2024-04-22T23:07:46.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:46.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:46.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:46.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:46.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:46.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: Adjusting osd_memory_target on smithi069 to 3216M 2024-04-22T23:07:46.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:46.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:46.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:07:46.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:46.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:07:46.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:46.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/3183468414' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67763500-c0be-4838-beb5-ea7f69c4395f"}]: dispatch 2024-04-22T23:07:46.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67763500-c0be-4838-beb5-ea7f69c4395f"}]: dispatch 2024-04-22T23:07:46.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "67763500-c0be-4838-beb5-ea7f69c4395f"}]': finished 2024-04-22T23:07:46.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T23:07:46.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:07:46.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:45 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/1167792688' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:07:46.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: Detected new or changed devices on smithi069 2024-04-22T23:07:46.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:46.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:46.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:46.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:46.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:07:46.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: Adjusting osd_memory_target on smithi069 to 3216M 2024-04-22T23:07:46.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:46.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:46.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:07:46.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:46.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:07:46.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:46.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='client.? 172.21.15.149:0/3183468414' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67763500-c0be-4838-beb5-ea7f69c4395f"}]: dispatch 2024-04-22T23:07:46.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67763500-c0be-4838-beb5-ea7f69c4395f"}]: dispatch 2024-04-22T23:07:46.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "67763500-c0be-4838-beb5-ea7f69c4395f"}]': finished 2024-04-22T23:07:46.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T23:07:46.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:07:46.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:45 smithi149 ceph-mon[23948]: from='client.? 172.21.15.149:0/1167792688' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:07:47.206 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:46 smithi149 ceph-mon[23948]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:47.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:46 smithi069 ceph-mon[21859]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:49.065 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:49 smithi149 ceph-mon[23948]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:49.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:49 smithi069 ceph-mon[21859]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:51.369 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:51 smithi149 ceph-mon[23948]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:51.370 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:51 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T23:07:51.370 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:51 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:51.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:51 smithi069 ceph-mon[21859]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:51.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:51 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T23:07:51.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:51 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:52.385 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:52 smithi149 ceph-mon[23948]: Deploying daemon osd.4 on smithi149 2024-04-22T23:07:52.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:52 smithi069 ceph-mon[21859]: Deploying daemon osd.4 on smithi149 2024-04-22T23:07:53.386 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:53 smithi149 ceph-mon[23948]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:53.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:53 smithi069 ceph-mon[21859]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:54.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:54 smithi149 ceph-mon[23948]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:54.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:54 smithi069 ceph-mon[21859]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:57.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:57.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:07:57.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:57.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:07:57.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:57 smithi149 ceph-mon[23948]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:57.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:07:57.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:07:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:07:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:07:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:57 smithi069 ceph-mon[21859]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:59.334 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:07:59 smithi149 ceph-mon[23948]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:59.343 INFO:teuthology.orchestra.run.smithi149.stdout:Created osd(s) 4 on host 'smithi149' 2024-04-22T23:07:59.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:07:59 smithi069 ceph-mon[21859]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:07:59.914 DEBUG:teuthology.orchestra.run.smithi149:osd.4> sudo journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.4.service 2024-04-22T23:07:59.917 INFO:tasks.cephadm:Deploying osd.5 on smithi149 with /dev/vg_nvme/lv_3... 2024-04-22T23:07:59.917 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-22T23:08:00.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:00 smithi149 ceph-mon[23948]: from='osd.4 [v2:172.21.15.149:6800/1064192336,v1:172.21.15.149:6801/1064192336]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T23:08:00.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:00 smithi149 ceph-mon[23948]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T23:08:00.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:00 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:00.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:00 smithi069 ceph-mon[21859]: from='osd.4 [v2:172.21.15.149:6800/1064192336,v1:172.21.15.149:6801/1064192336]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T23:08:00.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:00 smithi069 ceph-mon[21859]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T23:08:00.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:00 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:01.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:01 smithi149 ceph-mon[23948]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:08:01.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:01 smithi149 ceph-mon[23948]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T23:08:01.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:01 smithi149 ceph-mon[23948]: osdmap e30: 5 total, 4 up, 5 in 2024-04-22T23:08:01.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:01 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:08:01.379 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:01 smithi149 ceph-mon[23948]: from='osd.4 [v2:172.21.15.149:6800/1064192336,v1:172.21.15.149:6801/1064192336]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:08:01.379 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:01 smithi149 ceph-mon[23948]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:08:01.379 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:01 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:01.380 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:08:01 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[28172]: 2024-04-22T23:08:01.084+0000 7f6db3ec1700 -1 osd.4 0 waiting for initial osdmap 2024-04-22T23:08:01.380 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:08:01 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[28172]: 2024-04-22T23:08:01.089+0000 7f6dad055700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:08:01.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:01 smithi069 ceph-mon[21859]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:08:01.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:01 smithi069 ceph-mon[21859]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T23:08:01.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:01 smithi069 ceph-mon[21859]: osdmap e30: 5 total, 4 up, 5 in 2024-04-22T23:08:01.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:01 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:08:01.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:01 smithi069 ceph-mon[21859]: from='osd.4 [v2:172.21.15.149:6800/1064192336,v1:172.21.15.149:6801/1064192336]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:08:01.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:01 smithi069 ceph-mon[21859]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:08:01.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:01 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:02.379 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:02 smithi149 ceph-mon[23948]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-04-22T23:08:02.379 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:02 smithi149 ceph-mon[23948]: osdmap e31: 5 total, 4 up, 5 in 2024-04-22T23:08:02.379 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:02 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:08:02.380 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:02 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:08:02.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:02 smithi069 ceph-mon[21859]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-04-22T23:08:02.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:02 smithi069 ceph-mon[21859]: osdmap e31: 5 total, 4 up, 5 in 2024-04-22T23:08:02.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:02 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:08:02.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:02 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:08:03.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:03 smithi069 ceph-mon[21859]: purged_snaps scrub starts 2024-04-22T23:08:03.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:03 smithi069 ceph-mon[21859]: purged_snaps scrub ok 2024-04-22T23:08:03.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:03 smithi069 ceph-mon[21859]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:08:03.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:03 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:08:03.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:03 smithi069 ceph-mon[21859]: osd.4 [v2:172.21.15.149:6800/1064192336,v1:172.21.15.149:6801/1064192336] boot 2024-04-22T23:08:03.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:03 smithi069 ceph-mon[21859]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T23:08:03.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:03 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:08:03.399 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:03 smithi149 ceph-mon[23948]: purged_snaps scrub starts 2024-04-22T23:08:03.399 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:03 smithi149 ceph-mon[23948]: purged_snaps scrub ok 2024-04-22T23:08:03.399 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:03 smithi149 ceph-mon[23948]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T23:08:03.399 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:03 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:08:03.399 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:03 smithi149 ceph-mon[23948]: osd.4 [v2:172.21.15.149:6800/1064192336,v1:172.21.15.149:6801/1064192336] boot 2024-04-22T23:08:03.399 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:03 smithi149 ceph-mon[23948]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T23:08:03.399 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:03 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:08:03.692 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-22T23:08:03.692 INFO:teuthology.orchestra.run.smithi149.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-22T23:08:03.692 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 10+0 records in 2024-04-22T23:08:03.692 INFO:teuthology.orchestra.run.smithi149.stderr:10+0 records out 2024-04-22T23:08:03.692 INFO:teuthology.orchestra.run.smithi149.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.012538 s, 836 MB/s 2024-04-22T23:08:03.692 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping successful for: 2024-04-22T23:08:04.304 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:04 smithi149 ceph-mon[23948]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T23:08:04.328 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi149:vg_nvme/lv_3 2024-04-22T23:08:04.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:04 smithi069 ceph-mon[21859]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T23:08:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:05 smithi149 ceph-mon[23948]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T23:08:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:05 smithi149 ceph-mon[23948]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T23:08:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:05 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:05 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:08:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:05 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:05 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:05 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:08:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:05 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:05 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:08:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:05 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:05.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:05 smithi069 ceph-mon[21859]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T23:08:05.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:05 smithi069 ceph-mon[21859]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T23:08:05.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:05 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:05.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:05 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:08:05.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:05 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:05.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:05 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:05.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:05 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:08:05.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:05 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:05.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:05 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:08:05.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:05 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:06.375 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:06 smithi149 ceph-mon[23948]: Detected new or changed devices on smithi149 2024-04-22T23:08:06.375 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:06 smithi149 ceph-mon[23948]: Adjusting osd_memory_target on smithi149 to 4664M 2024-04-22T23:08:06.375 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:06 smithi149 ceph-mon[23948]: osdmap e35: 5 total, 5 up, 5 in 2024-04-22T23:08:06.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:06 smithi069 ceph-mon[21859]: Detected new or changed devices on smithi149 2024-04-22T23:08:06.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:06 smithi069 ceph-mon[21859]: Adjusting osd_memory_target on smithi149 to 4664M 2024-04-22T23:08:06.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:06 smithi069 ceph-mon[21859]: osdmap e35: 5 total, 5 up, 5 in 2024-04-22T23:08:07.384 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:07 smithi149 ceph-mon[23948]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T23:08:07.385 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:07 smithi149 ceph-mon[23948]: osdmap e36: 5 total, 5 up, 5 in 2024-04-22T23:08:07.385 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:07 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:08:07.385 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:07 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:08:07.385 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:07 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:07.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:07 smithi069 ceph-mon[21859]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T23:08:07.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:07 smithi069 ceph-mon[21859]: osdmap e36: 5 total, 5 up, 5 in 2024-04-22T23:08:07.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:07 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:08:07.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:07 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:08:07.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:07 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:08 smithi069 ceph-mon[21859]: from='client.24155 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:08:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:08:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:08:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:08 smithi149 ceph-mon[23948]: from='client.24155 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:08:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:08:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:08:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:09 smithi069 ceph-mon[21859]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-04-22T23:08:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:09 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/3203585049' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59adfbba-e1cf-461c-9698-6dd9755a808b"}]: dispatch 2024-04-22T23:08:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:09 smithi069 ceph-mon[21859]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59adfbba-e1cf-461c-9698-6dd9755a808b"}]: dispatch 2024-04-22T23:08:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:09 smithi069 ceph-mon[21859]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59adfbba-e1cf-461c-9698-6dd9755a808b"}]': finished 2024-04-22T23:08:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:09 smithi069 ceph-mon[21859]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T23:08:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:09 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:08:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:09 smithi149 ceph-mon[23948]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-04-22T23:08:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:09 smithi149 ceph-mon[23948]: from='client.? 172.21.15.149:0/3203585049' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59adfbba-e1cf-461c-9698-6dd9755a808b"}]: dispatch 2024-04-22T23:08:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:09 smithi149 ceph-mon[23948]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59adfbba-e1cf-461c-9698-6dd9755a808b"}]: dispatch 2024-04-22T23:08:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:09 smithi149 ceph-mon[23948]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59adfbba-e1cf-461c-9698-6dd9755a808b"}]': finished 2024-04-22T23:08:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:09 smithi149 ceph-mon[23948]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T23:08:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:09 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:08:10.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:10 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/3221114205' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:08:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:10 smithi149 ceph-mon[23948]: from='client.? 172.21.15.149:0/3221114205' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:08:11.383 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:11 smithi149 ceph-mon[23948]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-22T23:08:11.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:11 smithi069 ceph-mon[21859]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-22T23:08:13.291 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:13 smithi149 ceph-mon[23948]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T23:08:13.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:13 smithi069 ceph-mon[21859]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T23:08:14.645 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:14 smithi149 ceph-mon[23948]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T23:08:14.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:14 smithi069 ceph-mon[21859]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T23:08:15.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:15 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T23:08:15.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:15 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:15.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:15 smithi149 ceph-mon[23948]: Deploying daemon osd.5 on smithi149 2024-04-22T23:08:15.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:15 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T23:08:15.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:15 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:15.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:15 smithi069 ceph-mon[21859]: Deploying daemon osd.5 on smithi149 2024-04-22T23:08:16.634 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:16 smithi149 ceph-mon[23948]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T23:08:16.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:16 smithi069 ceph-mon[21859]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T23:08:19.165 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:19 smithi149 ceph-mon[23948]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T23:08:19.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:19 smithi069 ceph-mon[21859]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T23:08:21.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:20 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:21.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:20 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:08:21.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:20 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:21.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:20 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:08:21.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:20 smithi069 ceph-mon[21859]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T23:08:21.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:20 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:21.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:20 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:08:21.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:20 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:21.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:20 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:08:21.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:20 smithi149 ceph-mon[23948]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T23:08:23.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:23 smithi149 ceph-mon[23948]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T23:08:23.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:23 smithi149 ceph-mon[23948]: from='osd.5 [v2:172.21.15.149:6808/2676738694,v1:172.21.15.149:6809/2676738694]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T23:08:23.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:23 smithi149 ceph-mon[23948]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T23:08:23.252 INFO:teuthology.orchestra.run.smithi149.stdout:Created osd(s) 5 on host 'smithi149' 2024-04-22T23:08:23.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:23 smithi069 ceph-mon[21859]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T23:08:23.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:23 smithi069 ceph-mon[21859]: from='osd.5 [v2:172.21.15.149:6808/2676738694,v1:172.21.15.149:6809/2676738694]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T23:08:23.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:23 smithi069 ceph-mon[21859]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T23:08:24.214 DEBUG:teuthology.orchestra.run.smithi149:osd.5> sudo journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.5.service 2024-04-22T23:08:24.217 INFO:tasks.cephadm:Deploying osd.6 on smithi149 with /dev/vg_nvme/lv_2... 2024-04-22T23:08:24.217 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-22T23:08:24.237 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:24 smithi149 ceph-mon[23948]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T23:08:24.237 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:24 smithi149 ceph-mon[23948]: osdmap e38: 6 total, 5 up, 6 in 2024-04-22T23:08:24.237 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:24 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:08:24.237 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:24 smithi149 ceph-mon[23948]: from='osd.5 [v2:172.21.15.149:6808/2676738694,v1:172.21.15.149:6809/2676738694]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:08:24.237 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:24 smithi149 ceph-mon[23948]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:08:24.237 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:24 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:24.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:24 smithi069 ceph-mon[21859]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T23:08:24.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:24 smithi069 ceph-mon[21859]: osdmap e38: 6 total, 5 up, 6 in 2024-04-22T23:08:24.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:24 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:08:24.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:24 smithi069 ceph-mon[21859]: from='osd.5 [v2:172.21.15.149:6808/2676738694,v1:172.21.15.149:6809/2676738694]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:08:24.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:24 smithi069 ceph-mon[21859]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:08:24.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:24 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:25.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:25 smithi149 ceph-mon[23948]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T23:08:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:25 smithi149 ceph-mon[23948]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-04-22T23:08:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:25 smithi149 ceph-mon[23948]: osdmap e39: 6 total, 5 up, 6 in 2024-04-22T23:08:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:25 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:08:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:25 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:08:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:25 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:25.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:25 smithi069 ceph-mon[21859]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T23:08:25.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:25 smithi069 ceph-mon[21859]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-04-22T23:08:25.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:25 smithi069 ceph-mon[21859]: osdmap e39: 6 total, 5 up, 6 in 2024-04-22T23:08:25.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:25 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:08:25.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:25 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:08:25.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:25 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:26.092 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:26 smithi149 ceph-mon[23948]: purged_snaps scrub starts 2024-04-22T23:08:26.092 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:26 smithi149 ceph-mon[23948]: purged_snaps scrub ok 2024-04-22T23:08:26.093 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:26 smithi149 ceph-mon[23948]: osd.5 [v2:172.21.15.149:6808/2676738694,v1:172.21.15.149:6809/2676738694] boot 2024-04-22T23:08:26.093 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:26 smithi149 ceph-mon[23948]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T23:08:26.093 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:26 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:08:26.285 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-22T23:08:26.285 INFO:teuthology.orchestra.run.smithi149.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-22T23:08:26.285 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 10+0 records in 2024-04-22T23:08:26.285 INFO:teuthology.orchestra.run.smithi149.stderr:10+0 records out 2024-04-22T23:08:26.286 INFO:teuthology.orchestra.run.smithi149.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0123618 s, 848 MB/s 2024-04-22T23:08:26.286 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping successful for: 2024-04-22T23:08:26.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:26 smithi069 ceph-mon[21859]: purged_snaps scrub starts 2024-04-22T23:08:26.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:26 smithi069 ceph-mon[21859]: purged_snaps scrub ok 2024-04-22T23:08:26.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:26 smithi069 ceph-mon[21859]: osd.5 [v2:172.21.15.149:6808/2676738694,v1:172.21.15.149:6809/2676738694] boot 2024-04-22T23:08:26.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:26 smithi069 ceph-mon[21859]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T23:08:26.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:26 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:08:26.890 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi149:vg_nvme/lv_2 2024-04-22T23:08:27.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:27 smithi149 ceph-mon[23948]: pgmap v127: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:27.118 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:27 smithi149 ceph-mon[23948]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T23:08:27.119 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:27 smithi149 ceph-mon[23948]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T23:08:27.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:27 smithi069 ceph-mon[21859]: pgmap v127: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:27.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:27 smithi069 ceph-mon[21859]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T23:08:27.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:27 smithi069 ceph-mon[21859]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T23:08:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:28 smithi149 ceph-mon[23948]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T23:08:28.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:28 smithi069 ceph-mon[21859]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T23:08:29.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:29 smithi069 ceph-mon[21859]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:29.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:29 smithi069 ceph-mon[21859]: osdmap e43: 6 total, 6 up, 6 in 2024-04-22T23:08:29.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:29 smithi149 ceph-mon[23948]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:29.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:29 smithi149 ceph-mon[23948]: osdmap e43: 6 total, 6 up, 6 in 2024-04-22T23:08:30.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:30 smithi149 ceph-mon[23948]: osdmap e44: 6 total, 6 up, 6 in 2024-04-22T23:08:30.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:30 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:30.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:30 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:08:30.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:30 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:08:30.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:30 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:30.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:30 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:30.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:30 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:08:30.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:30 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:30.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:30 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:08:30.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:30 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:30.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:30 smithi069 ceph-mon[21859]: osdmap e44: 6 total, 6 up, 6 in 2024-04-22T23:08:30.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:30 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:30.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:30 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:08:30.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:30 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:08:30.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:30 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:30.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:30 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:30.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:30 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:08:30.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:30 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:30.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:30 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:08:30.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:30 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:31 smithi149 ceph-mon[23948]: Detected new or changed devices on smithi149 2024-04-22T23:08:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:31 smithi149 ceph-mon[23948]: Adjusting osd_memory_target on smithi149 to 2332M 2024-04-22T23:08:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:31 smithi149 ceph-mon[23948]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:31 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:08:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:31 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:08:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:31 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:31.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:31 smithi069 ceph-mon[21859]: Detected new or changed devices on smithi149 2024-04-22T23:08:31.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:31 smithi069 ceph-mon[21859]: Adjusting osd_memory_target on smithi149 to 2332M 2024-04-22T23:08:31.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:31 smithi069 ceph-mon[21859]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:31.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:31 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:08:31.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:31 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:08:31.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:31 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:32.138 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:32 smithi149 ceph-mon[23948]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:08:32.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:32 smithi069 ceph-mon[21859]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:08:33.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:33 smithi069 ceph-mon[21859]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-22T23:08:33.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:33 smithi069 ceph-mon[21859]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T23:08:33.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:33 smithi069 ceph-mon[21859]: Cluster is now healthy 2024-04-22T23:08:33.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:33 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/4287420025' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "265277c2-f8b1-46e4-b9b1-c6b37de45487"}]: dispatch 2024-04-22T23:08:33.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:33 smithi069 ceph-mon[21859]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "265277c2-f8b1-46e4-b9b1-c6b37de45487"}]: dispatch 2024-04-22T23:08:33.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:33 smithi069 ceph-mon[21859]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "265277c2-f8b1-46e4-b9b1-c6b37de45487"}]': finished 2024-04-22T23:08:33.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:33 smithi069 ceph-mon[21859]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T23:08:33.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:08:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:33 smithi149 ceph-mon[23948]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-22T23:08:33.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:33 smithi149 ceph-mon[23948]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T23:08:33.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:33 smithi149 ceph-mon[23948]: Cluster is now healthy 2024-04-22T23:08:33.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:33 smithi149 ceph-mon[23948]: from='client.? 172.21.15.149:0/4287420025' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "265277c2-f8b1-46e4-b9b1-c6b37de45487"}]: dispatch 2024-04-22T23:08:33.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:33 smithi149 ceph-mon[23948]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "265277c2-f8b1-46e4-b9b1-c6b37de45487"}]: dispatch 2024-04-22T23:08:33.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:33 smithi149 ceph-mon[23948]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "265277c2-f8b1-46e4-b9b1-c6b37de45487"}]': finished 2024-04-22T23:08:33.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:33 smithi149 ceph-mon[23948]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T23:08:33.462 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:08:34.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:34 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/786253415' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:08:34.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:34 smithi149 ceph-mon[23948]: from='client.? 172.21.15.149:0/786253415' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:08:35.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:35 smithi069 ceph-mon[21859]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T23:08:35.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:35 smithi149 ceph-mon[23948]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T23:08:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:37 smithi149 ceph-mon[23948]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T23:08:37.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:37 smithi069 ceph-mon[21859]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T23:08:39.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:39 smithi069 ceph-mon[21859]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-22T23:08:39.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:39 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T23:08:39.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:39 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:39.397 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:39 smithi149 ceph-mon[23948]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-22T23:08:39.397 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:39 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T23:08:39.397 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:39 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:40.389 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:40 smithi149 ceph-mon[23948]: Deploying daemon osd.6 on smithi149 2024-04-22T23:08:40.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:40 smithi069 ceph-mon[21859]: Deploying daemon osd.6 on smithi149 2024-04-22T23:08:41.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:41 smithi069 ceph-mon[21859]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T23:08:41.448 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:41 smithi149 ceph-mon[23948]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T23:08:43.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:43 smithi149 ceph-mon[23948]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:43.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:43 smithi069 ceph-mon[21859]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:45 smithi149 ceph-mon[23948]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:08:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:45 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:08:45.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:45 smithi069 ceph-mon[21859]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:45.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:45.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:08:45.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:45.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:45 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:08:47.390 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:47 smithi149 ceph-mon[23948]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:47.425 INFO:teuthology.orchestra.run.smithi149.stdout:Created osd(s) 6 on host 'smithi149' 2024-04-22T23:08:47.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:47 smithi069 ceph-mon[21859]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:48.045 DEBUG:teuthology.orchestra.run.smithi149:osd.6> sudo journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.6.service 2024-04-22T23:08:48.048 INFO:tasks.cephadm:Deploying osd.7 on smithi149 with /dev/vg_nvme/lv_1... 2024-04-22T23:08:48.048 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-22T23:08:48.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:48 smithi149 ceph-mon[23948]: from='osd.6 [v2:172.21.15.149:6816/3434980700,v1:172.21.15.149:6817/3434980700]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T23:08:48.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:48 smithi149 ceph-mon[23948]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T23:08:48.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:48 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:48.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:48 smithi069 ceph-mon[21859]: from='osd.6 [v2:172.21.15.149:6816/3434980700,v1:172.21.15.149:6817/3434980700]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T23:08:48.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:48 smithi069 ceph-mon[21859]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T23:08:48.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:48 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:49.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:49 smithi149 ceph-mon[23948]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:49.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:49 smithi149 ceph-mon[23948]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T23:08:49.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:49 smithi149 ceph-mon[23948]: osdmap e46: 7 total, 6 up, 7 in 2024-04-22T23:08:49.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:49 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:08:49.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:49 smithi149 ceph-mon[23948]: from='osd.6 [v2:172.21.15.149:6816/3434980700,v1:172.21.15.149:6817/3434980700]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:08:49.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:49 smithi149 ceph-mon[23948]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:08:49.451 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:08:49 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[34194]: 2024-04-22T23:08:49.209+0000 7fb23b45e700 -1 osd.6 0 waiting for initial osdmap 2024-04-22T23:08:49.451 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:08:49 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[34194]: 2024-04-22T23:08:49.215+0000 7fb235df5700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:08:49.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:49 smithi069 ceph-mon[21859]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:49.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:49 smithi069 ceph-mon[21859]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T23:08:49.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:49 smithi069 ceph-mon[21859]: osdmap e46: 7 total, 6 up, 7 in 2024-04-22T23:08:49.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:49 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:08:49.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:49 smithi069 ceph-mon[21859]: from='osd.6 [v2:172.21.15.149:6816/3434980700,v1:172.21.15.149:6817/3434980700]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:08:49.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:49 smithi069 ceph-mon[21859]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:08:50.343 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:50 smithi149 ceph-mon[23948]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-04-22T23:08:50.343 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:50 smithi149 ceph-mon[23948]: osdmap e47: 7 total, 6 up, 7 in 2024-04-22T23:08:50.343 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:50 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:08:50.343 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:50 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:08:50.580 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-22T23:08:50.580 INFO:teuthology.orchestra.run.smithi149.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-22T23:08:50.580 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 10+0 records in 2024-04-22T23:08:50.580 INFO:teuthology.orchestra.run.smithi149.stderr:10+0 records out 2024-04-22T23:08:50.580 INFO:teuthology.orchestra.run.smithi149.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0127173 s, 825 MB/s 2024-04-22T23:08:50.580 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping successful for: 2024-04-22T23:08:50.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:50 smithi069 ceph-mon[21859]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-04-22T23:08:50.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:50 smithi069 ceph-mon[21859]: osdmap e47: 7 total, 6 up, 7 in 2024-04-22T23:08:50.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:50 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:08:50.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:50 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:08:51.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:51 smithi069 ceph-mon[21859]: purged_snaps scrub starts 2024-04-22T23:08:51.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:51 smithi069 ceph-mon[21859]: purged_snaps scrub ok 2024-04-22T23:08:51.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:51 smithi069 ceph-mon[21859]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:51.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:51 smithi069 ceph-mon[21859]: osd.6 [v2:172.21.15.149:6816/3434980700,v1:172.21.15.149:6817/3434980700] boot 2024-04-22T23:08:51.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:51 smithi069 ceph-mon[21859]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T23:08:51.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:51 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:08:51.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:51 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:51.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:51 smithi149 ceph-mon[23948]: purged_snaps scrub starts 2024-04-22T23:08:51.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:51 smithi149 ceph-mon[23948]: purged_snaps scrub ok 2024-04-22T23:08:51.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:51 smithi149 ceph-mon[23948]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T23:08:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:51 smithi149 ceph-mon[23948]: osd.6 [v2:172.21.15.149:6816/3434980700,v1:172.21.15.149:6817/3434980700] boot 2024-04-22T23:08:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:51 smithi149 ceph-mon[23948]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T23:08:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:51 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:08:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:51 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:51.926 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi149:vg_nvme/lv_1 2024-04-22T23:08:52.480 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:52 smithi149 ceph-mon[23948]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T23:08:52.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:52 smithi069 ceph-mon[21859]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T23:08:53.425 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:53 smithi149 ceph-mon[23948]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:08:53.425 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:53 smithi149 ceph-mon[23948]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T23:08:53.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:53 smithi069 ceph-mon[21859]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:08:53.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:53 smithi069 ceph-mon[21859]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T23:08:54.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:54 smithi149 ceph-mon[23948]: osdmap e51: 7 total, 7 up, 7 in 2024-04-22T23:08:54.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:08:54.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:08:54.323 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:54.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:54 smithi069 ceph-mon[21859]: osdmap e51: 7 total, 7 up, 7 in 2024-04-22T23:08:54.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T23:08:54.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T23:08:54.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:55.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:55 smithi069 ceph-mon[21859]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:08:55.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:55 smithi069 ceph-mon[21859]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:08:55.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:55 smithi069 ceph-mon[21859]: osdmap e52: 7 total, 7 up, 7 in 2024-04-22T23:08:55.647 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:55 smithi149 ceph-mon[23948]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:08:55.647 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:55 smithi149 ceph-mon[23948]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:08:55.647 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:55 smithi149 ceph-mon[23948]: osdmap e52: 7 total, 7 up, 7 in 2024-04-22T23:08:57.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:08:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: Detected new or changed devices on smithi149 2024-04-22T23:08:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:08:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:08:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:08:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: Adjusting osd_memory_target on smithi149 to 1554M 2024-04-22T23:08:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:08:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:08:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:57 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:08:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: Detected new or changed devices on smithi149 2024-04-22T23:08:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:08:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:08:57.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:08:57.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: Adjusting osd_memory_target on smithi149 to 1554M 2024-04-22T23:08:57.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:57.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:57.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:08:57.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:08:57.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:08:57.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:57 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:08:58.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:58 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/1653939244' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e49fc3e2-0333-4c4e-9edf-d85933fd46c0"}]: dispatch 2024-04-22T23:08:58.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:58 smithi069 ceph-mon[21859]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e49fc3e2-0333-4c4e-9edf-d85933fd46c0"}]: dispatch 2024-04-22T23:08:58.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:58 smithi069 ceph-mon[21859]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e49fc3e2-0333-4c4e-9edf-d85933fd46c0"}]': finished 2024-04-22T23:08:58.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:58 smithi069 ceph-mon[21859]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T23:08:58.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:58 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:08:58.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:58 smithi069 ceph-mon[21859]: from='client.? 172.21.15.149:0/2954391012' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:08:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:58 smithi149 ceph-mon[23948]: from='client.? 172.21.15.149:0/1653939244' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e49fc3e2-0333-4c4e-9edf-d85933fd46c0"}]: dispatch 2024-04-22T23:08:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:58 smithi149 ceph-mon[23948]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e49fc3e2-0333-4c4e-9edf-d85933fd46c0"}]: dispatch 2024-04-22T23:08:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:58 smithi149 ceph-mon[23948]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e49fc3e2-0333-4c4e-9edf-d85933fd46c0"}]': finished 2024-04-22T23:08:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:58 smithi149 ceph-mon[23948]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T23:08:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:58 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:08:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:58 smithi149 ceph-mon[23948]: from='client.? 172.21.15.149:0/2954391012' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T23:08:59.398 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:08:59 smithi069 ceph-mon[21859]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-04-22T23:08:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:08:59 smithi149 ceph-mon[23948]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-04-22T23:09:01.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:01 smithi149 ceph-mon[23948]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-04-22T23:09:01.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:01 smithi069 ceph-mon[21859]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-04-22T23:09:03.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:03 smithi069 ceph-mon[21859]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T23:09:03.397 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:03 smithi149 ceph-mon[23948]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T23:09:04.372 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:04 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T23:09:04.373 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:04 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:09:04.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:04 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T23:09:04.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:04 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:09:05.268 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:05 smithi149 ceph-mon[23948]: Deploying daemon osd.7 on smithi149 2024-04-22T23:09:05.268 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:05 smithi149 ceph-mon[23948]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-04-22T23:09:05.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:05 smithi069 ceph-mon[21859]: Deploying daemon osd.7 on smithi149 2024-04-22T23:09:05.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:05 smithi069 ceph-mon[21859]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-04-22T23:09:07.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:07 smithi069 ceph-mon[21859]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T23:09:07.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:07 smithi149 ceph-mon[23948]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T23:09:08.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:09:08.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:09:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:09:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:09:09.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:09 smithi069 ceph-mon[21859]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:09:09.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:09 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:09 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:09:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:09 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:09:09.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:09 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:09:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:09 smithi149 ceph-mon[23948]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:09:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:09 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:09 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:09:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:09 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:09:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:09 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:09:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:11 smithi149 ceph-mon[23948]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:09:11.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:11 smithi069 ceph-mon[21859]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:09:11.795 INFO:teuthology.orchestra.run.smithi149.stdout:Created osd(s) 7 on host 'smithi149' 2024-04-22T23:09:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:12 smithi149 ceph-mon[23948]: from='osd.7 [v2:172.21.15.149:6824/1513400132,v1:172.21.15.149:6825/1513400132]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T23:09:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:12 smithi149 ceph-mon[23948]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T23:09:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:12 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:12.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:12 smithi069 ceph-mon[21859]: from='osd.7 [v2:172.21.15.149:6824/1513400132,v1:172.21.15.149:6825/1513400132]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T23:09:12.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:12 smithi069 ceph-mon[21859]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T23:09:12.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:12 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:12.820 DEBUG:teuthology.orchestra.run.smithi149:osd.7> sudo journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.7.service 2024-04-22T23:09:12.822 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-22T23:09:12.823 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-22T23:09:13.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:13 smithi069 ceph-mon[21859]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:09:13.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:13 smithi069 ceph-mon[21859]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T23:09:13.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:13 smithi069 ceph-mon[21859]: osdmap e54: 8 total, 7 up, 8 in 2024-04-22T23:09:13.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:13 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:09:13.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:13 smithi069 ceph-mon[21859]: from='osd.7 [v2:172.21.15.149:6824/1513400132,v1:172.21.15.149:6825/1513400132]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:09:13.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:13 smithi069 ceph-mon[21859]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:09:13.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:13 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:13.445 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:13 smithi149 ceph-mon[23948]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:09:13.446 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:13 smithi149 ceph-mon[23948]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T23:09:13.446 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:13 smithi149 ceph-mon[23948]: osdmap e54: 8 total, 7 up, 8 in 2024-04-22T23:09:13.446 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:13 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:09:13.446 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:13 smithi149 ceph-mon[23948]: from='osd.7 [v2:172.21.15.149:6824/1513400132,v1:172.21.15.149:6825/1513400132]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:09:13.446 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:13 smithi149 ceph-mon[23948]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:09:13.446 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:13 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:13.446 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:09:13 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[37197]: 2024-04-22T23:09:13.163+0000 7f4a4eb5b700 -1 osd.7 0 waiting for initial osdmap 2024-04-22T23:09:13.446 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:09:13 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[37197]: 2024-04-22T23:09:13.170+0000 7f4a4acf5700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:09:14.274 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:14 smithi149 ceph-mon[23948]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-04-22T23:09:14.275 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:14 smithi149 ceph-mon[23948]: osdmap e55: 8 total, 7 up, 8 in 2024-04-22T23:09:14.275 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:14 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:09:14.275 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:14 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:09:14.275 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:14 smithi149 ceph-mon[23948]: osd.7 [v2:172.21.15.149:6824/1513400132,v1:172.21.15.149:6825/1513400132] boot 2024-04-22T23:09:14.275 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:14 smithi149 ceph-mon[23948]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T23:09:14.275 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:14 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:09:14.467 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:14 smithi069 ceph-mon[21859]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2024-04-22T23:09:14.467 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:14 smithi069 ceph-mon[21859]: osdmap e55: 8 total, 7 up, 8 in 2024-04-22T23:09:14.467 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:09:14.467 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:09:14.468 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:14 smithi069 ceph-mon[21859]: osd.7 [v2:172.21.15.149:6824/1513400132,v1:172.21.15.149:6825/1513400132] boot 2024-04-22T23:09:14.468 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:14 smithi069 ceph-mon[21859]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T23:09:14.468 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:14 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:09:14.855 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:09:15.269 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:15 smithi149 ceph-mon[23948]: purged_snaps scrub starts 2024-04-22T23:09:15.269 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:15 smithi149 ceph-mon[23948]: purged_snaps scrub ok 2024-04-22T23:09:15.269 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:15 smithi149 ceph-mon[23948]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:09:15.269 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:15 smithi149 ceph-mon[23948]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T23:09:15.269 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:15 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/3446149381' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T23:09:15.585 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:15 smithi069 ceph-mon[21859]: purged_snaps scrub starts 2024-04-22T23:09:15.585 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:15 smithi069 ceph-mon[21859]: purged_snaps scrub ok 2024-04-22T23:09:15.585 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:15 smithi069 ceph-mon[21859]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-22T23:09:15.585 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:15 smithi069 ceph-mon[21859]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T23:09:15.585 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:15 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3446149381' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T23:09:15.609 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1713827353,"num_in_osds":8,"osd_in_since":1713827337,"num_remapped_pgs":1} 2024-04-22T23:09:15.609 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T23:09:15.830 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:16.789 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T23:09:16.789 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: Detected new or changed devices on smithi149 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: Adjusting osd_memory_target on smithi149 to 1166M 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:16.790 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:16 smithi069 ceph-mon[21859]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:16.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T23:09:16.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: Detected new or changed devices on smithi149 2024-04-22T23:09:16.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:16.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:09:16.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:09:16.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:09:16.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:09:16.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: Adjusting osd_memory_target on smithi149 to 1166M 2024-04-22T23:09:16.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:16.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:16.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:09:16.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:09:16.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:09:16.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:09:16.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:16 smithi149 ceph-mon[23948]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:17.607 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:09:17.607 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":60,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","created":"2024-04-22T23:03:17.786186+0000","modified":"2024-04-22T23:09:17.463867+0000","last_up_change":"2024-04-22T23:09:13.448317+0000","last_in_change":"2024-04-22T23:08:57.123079+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T23:07:16.294432+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"7d98e167-3473-493e-8deb-1e10412bf5ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6802","nonce":2446576616},{"type":"v1","addr":"172.21.15.69:6803","nonce":2446576616}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6804","nonce":2446576616},{"type":"v1","addr":"172.21.15.69:6805","nonce":2446576616}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6808","nonce":2446576616},{"type":"v1","addr":"172.21.15.69:6809","nonce":2446576616}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6806","nonce":2446576616},{"type":"v1","addr":"172.21.15.69:6807","nonce":2446576616}]},"public_addr":"172.21.15.69:6803/2446576616","cluster_addr":"172.21.15.69:6805/2446576616","heartbeat_back_addr":"172.21.15.69:6809/2446576616","heartbeat_front_addr":"172.21.15.69:6807/2446576616","state":["exists","up"]},{"osd":1,"uuid":"9d15522c-2ca4-451c-a569-f31e25e5a08b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6810","nonce":2958036688},{"type":"v1","addr":"172.21.15.69:6811","nonce":2958036688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6812","nonce":2958036688},{"type":"v1","addr":"172.21.15.69:6813","nonce":2958036688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6816","nonce":2958036688},{"type":"v1","addr":"172.21.15.69:6817","nonce":2958036688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6814","nonce":2958036688},{"type":"v1","addr":"172.21.15.69:6815","nonce":2958036688}]},"public_addr":"172.21.15.69:6811/2958036688","cluster_addr":"172.21.15.69:6813/2958036688","heartbeat_back_addr":"172.21.15.69:6817/2958036688","heartbeat_front_addr":"172.21.15.69:6815/2958036688","state":["exists","up"]},{"osd":2,"uuid":"14d41cfc-5bca-40a5-bede-a02925fa4cb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6818","nonce":2823169129},{"type":"v1","addr":"172.21.15.69:6819","nonce":2823169129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6820","nonce":2823169129},{"type":"v1","addr":"172.21.15.69:6821","nonce":2823169129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6824","nonce":2823169129},{"type":"v1","addr":"172.21.15.69:6825","nonce":2823169129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6822","nonce":2823169129},{"type":"v1","addr":"172.21.15.69:6823","nonce":2823169129}]},"public_addr":"172.21.15.69:6819/2823169129","cluster_addr":"172.21.15.69:6821/2823169129","heartbeat_back_addr":"172.21.15.69:6825/2823169129","heartbeat_front_addr":"172.21.15.69:6823/2823169129","state":["exists","up"]},{"osd":3,"uuid":"40b632b4-3632-4281-bde9-e7aeadecf33f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6826","nonce":1031635705},{"type":"v1","addr":"172.21.15.69:6827","nonce":1031635705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6828","nonce":1031635705},{"type":"v1","addr":"172.21.15.69:6829","nonce":1031635705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6832","nonce":1031635705},{"type":"v1","addr":"172.21.15.69:6833","nonce":1031635705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6830","nonce":1031635705},{"type":"v1","addr":"172.21.15.69:6831","nonce":1031635705}]},"public_addr":"172.21.15.69:6827/1031635705","cluster_addr":"172.21.15.69:6829/1031635705","heartbeat_back_addr":"172.21.15.69:6833/1031635705","heartbeat_front_addr":"172.21.15.69:6831/1031635705","state":["exists","up"]},{"osd":4,"uuid":"67763500-c0be-4838-beb5-ea7f69c4395f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":1064192336},{"type":"v1","addr":"172.21.15.149:6801","nonce":1064192336}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":1064192336},{"type":"v1","addr":"172.21.15.149:6803","nonce":1064192336}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":1064192336},{"type":"v1","addr":"172.21.15.149:6807","nonce":1064192336}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":1064192336},{"type":"v1","addr":"172.21.15.149:6805","nonce":1064192336}]},"public_addr":"172.21.15.149:6801/1064192336","cluster_addr":"172.21.15.149:6803/1064192336","heartbeat_back_addr":"172.21.15.149:6807/1064192336","heartbeat_front_addr":"172.21.15.149:6805/1064192336","state":["exists","up"]},{"osd":5,"uuid":"59adfbba-e1cf-461c-9698-6dd9755a808b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2676738694},{"type":"v1","addr":"172.21.15.149:6809","nonce":2676738694}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2676738694},{"type":"v1","addr":"172.21.15.149:6811","nonce":2676738694}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2676738694},{"type":"v1","addr":"172.21.15.149:6815","nonce":2676738694}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2676738694},{"type":"v1","addr":"172.21.15.149:6813","nonce":2676738694}]},"public_addr":"172.21.15.149:6809/2676738694","cluster_addr":"172.21.15.149:6811/2676738694","heartbeat_back_addr":"172.21.15.149:6815/2676738694","heartbeat_front_addr":"172.21.15.149:6813/2676738694","state":["exists","up"]},{"osd":6,"uuid":"265277c2-f8b1-46e4-b9b1-c6b37de45487","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":3434980700},{"type":"v1","addr":"172.21.15.149:6817","nonce":3434980700}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":3434980700},{"type":"v1","addr":"172.21.15.149:6819","nonce":3434980700}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":3434980700},{"type":"v1","addr":"172.21.15.149:6823","nonce":3434980700}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":3434980700},{"type":"v1","addr":"172.21.15.149:6821","nonce":3434980700}]},"public_addr":"172.21.15.149:6817/3434980700","cluster_addr":"172.21.15.149:6819/3434980700","heartbeat_back_addr":"172.21.15.149:6823/3434980700","heartbeat_front_addr":"172.21.15.149:6821/3434980700","state":["exists","up"]},{"osd":7,"uuid":"e49fc3e2-0333-4c4e-9edf-d85933fd46c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":1513400132},{"type":"v1","addr":"172.21.15.149:6825","nonce":1513400132}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":1513400132},{"type":"v1","addr":"172.21.15.149:6827","nonce":1513400132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":1513400132},{"type":"v1","addr":"172.21.15.149:6831","nonce":1513400132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":1513400132},{"type":"v1","addr":"172.21.15.149:6829","nonce":1513400132}]},"public_addr":"172.21.15.149:6825/1513400132","cluster_addr":"172.21.15.149:6827/1513400132","heartbeat_back_addr":"172.21.15.149:6831/1513400132","heartbeat_front_addr":"172.21.15.149:6829/1513400132","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:06:12.023674+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:06:45.135845+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:07:12.113109+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:07:38.761143+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:08:00.097037+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:08:23.879296+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:08:48.237571+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:09:12.503129+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.69:6800/2237930990":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/2087225558":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/2589858391":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/3578465763":"2024-04-23T23:04:13.845918+0000","172.21.15.69:6801/745479619":"2024-04-23T23:04:13.845918+0000","172.21.15.69:0/849543113":"2024-04-23T23:04:13.845918+0000","172.21.15.69:6800/745479619":"2024-04-23T23:04:13.845918+0000","172.21.15.69:0/3147401358":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/4144860677":"2024-04-23T23:03:44.182966+0000","172.21.15.69:0/2445874285":"2024-04-23T23:03:44.182966+0000","172.21.15.69:6801/2237930990":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/1041575105":"2024-04-23T23:04:13.845918+0000","172.21.15.69:0/493865482":"2024-04-23T23:03:44.182966+0000","172.21.15.69:6800/3986083499":"2024-04-23T23:03:44.182966+0000","172.21.15.69:6801/3986083499":"2024-04-23T23:03:44.182966+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T23:09:17.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:17 smithi069 ceph-mon[21859]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T23:09:17.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:17 smithi149 ceph-mon[23948]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T23:09:18.257 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-22T23:07:16.294432+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-22T23:09:18.258 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-22T23:09:18.472 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:18.802 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:18 smithi069 ceph-mon[21859]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T23:09:18.802 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:18 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1902808241' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T23:09:18.802 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:18 smithi069 ceph-mon[21859]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:18.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:18 smithi149 ceph-mon[23948]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T23:09:18.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:18 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/1902808241' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T23:09:18.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:18 smithi149 ceph-mon[23948]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:20.229 INFO:teuthology.orchestra.run.smithi069.stdout:pg_num: 1 2024-04-22T23:09:20.995 INFO:tasks.cephadm:Setting up client nodes... 2024-04-22T23:09:20.996 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-22T23:09:21.301 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:21 smithi069 ceph-mon[21859]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-22T23:09:21.302 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:21 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2216323358' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T23:09:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:21 smithi149 ceph-mon[23948]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-22T23:09:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:21 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2216323358' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T23:09:23.044 INFO:teuthology.orchestra.run.smithi069.stdout:[client.0] 2024-04-22T23:09:23.045 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQAj7iZmIXtsAhAAmMUhEDln92JQWdIJqpUExQ== 2024-04-22T23:09:23.392 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:23 smithi069 ceph-mon[21859]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-04-22T23:09:23.392 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:23 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1697282881' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T23:09:23.392 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:23 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1697282881' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T23:09:23.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:23 smithi149 ceph-mon[23948]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-04-22T23:09:23.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:23 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/1697282881' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T23:09:23.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:23 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/1697282881' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T23:09:23.733 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-22T23:09:23.733 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-22T23:09:23.733 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-22T23:09:23.768 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-22T23:09:23.768 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-22T23:09:23.768 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-22T23:09:24.013 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:24.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:24 smithi069 ceph-mon[21859]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T23:09:24.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:24 smithi149 ceph-mon[23948]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T23:09:25.838 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:09:26.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:25 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/672302099' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T23:09:26.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:25 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/672302099' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T23:09:26.755 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":19,"active_gid":14204,"active_name":"smithi069.lyiiqg","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6800","nonce":4253356569},{"type":"v1","addr":"172.21.15.69:6801","nonce":4253356569}]},"active_addr":"172.21.15.69:6801/4253356569","active_change":"2024-04-22T23:05:07.221865+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14222,"name":"smithi149.rxqkqk","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.69:8443/","prometheus":"http://172.21.15.69:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.69:0","nonce":4125178162}]},{"addrvec":[{"type":"v2","addr":"172.21.15.69:0","nonce":1631667929}]},{"addrvec":[{"type":"v2","addr":"172.21.15.69:0","nonce":3618686422}]},{"addrvec":[{"type":"v2","addr":"172.21.15.69:0","nonce":73619189}]}]}} 2024-04-22T23:09:26.761 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-22T23:09:26.761 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-22T23:09:26.761 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T23:09:26.991 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:27.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:26 smithi069 ceph-mon[21859]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-22T23:09:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:26 smithi149 ceph-mon[23948]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-22T23:09:28.728 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:09:28.728 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":60,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","created":"2024-04-22T23:03:17.786186+0000","modified":"2024-04-22T23:09:17.463867+0000","last_up_change":"2024-04-22T23:09:13.448317+0000","last_in_change":"2024-04-22T23:08:57.123079+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T23:07:16.294432+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"7d98e167-3473-493e-8deb-1e10412bf5ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6802","nonce":2446576616},{"type":"v1","addr":"172.21.15.69:6803","nonce":2446576616}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6804","nonce":2446576616},{"type":"v1","addr":"172.21.15.69:6805","nonce":2446576616}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6808","nonce":2446576616},{"type":"v1","addr":"172.21.15.69:6809","nonce":2446576616}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6806","nonce":2446576616},{"type":"v1","addr":"172.21.15.69:6807","nonce":2446576616}]},"public_addr":"172.21.15.69:6803/2446576616","cluster_addr":"172.21.15.69:6805/2446576616","heartbeat_back_addr":"172.21.15.69:6809/2446576616","heartbeat_front_addr":"172.21.15.69:6807/2446576616","state":["exists","up"]},{"osd":1,"uuid":"9d15522c-2ca4-451c-a569-f31e25e5a08b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6810","nonce":2958036688},{"type":"v1","addr":"172.21.15.69:6811","nonce":2958036688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6812","nonce":2958036688},{"type":"v1","addr":"172.21.15.69:6813","nonce":2958036688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6816","nonce":2958036688},{"type":"v1","addr":"172.21.15.69:6817","nonce":2958036688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6814","nonce":2958036688},{"type":"v1","addr":"172.21.15.69:6815","nonce":2958036688}]},"public_addr":"172.21.15.69:6811/2958036688","cluster_addr":"172.21.15.69:6813/2958036688","heartbeat_back_addr":"172.21.15.69:6817/2958036688","heartbeat_front_addr":"172.21.15.69:6815/2958036688","state":["exists","up"]},{"osd":2,"uuid":"14d41cfc-5bca-40a5-bede-a02925fa4cb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6818","nonce":2823169129},{"type":"v1","addr":"172.21.15.69:6819","nonce":2823169129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6820","nonce":2823169129},{"type":"v1","addr":"172.21.15.69:6821","nonce":2823169129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6824","nonce":2823169129},{"type":"v1","addr":"172.21.15.69:6825","nonce":2823169129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6822","nonce":2823169129},{"type":"v1","addr":"172.21.15.69:6823","nonce":2823169129}]},"public_addr":"172.21.15.69:6819/2823169129","cluster_addr":"172.21.15.69:6821/2823169129","heartbeat_back_addr":"172.21.15.69:6825/2823169129","heartbeat_front_addr":"172.21.15.69:6823/2823169129","state":["exists","up"]},{"osd":3,"uuid":"40b632b4-3632-4281-bde9-e7aeadecf33f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6826","nonce":1031635705},{"type":"v1","addr":"172.21.15.69:6827","nonce":1031635705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6828","nonce":1031635705},{"type":"v1","addr":"172.21.15.69:6829","nonce":1031635705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6832","nonce":1031635705},{"type":"v1","addr":"172.21.15.69:6833","nonce":1031635705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6830","nonce":1031635705},{"type":"v1","addr":"172.21.15.69:6831","nonce":1031635705}]},"public_addr":"172.21.15.69:6827/1031635705","cluster_addr":"172.21.15.69:6829/1031635705","heartbeat_back_addr":"172.21.15.69:6833/1031635705","heartbeat_front_addr":"172.21.15.69:6831/1031635705","state":["exists","up"]},{"osd":4,"uuid":"67763500-c0be-4838-beb5-ea7f69c4395f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":1064192336},{"type":"v1","addr":"172.21.15.149:6801","nonce":1064192336}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":1064192336},{"type":"v1","addr":"172.21.15.149:6803","nonce":1064192336}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":1064192336},{"type":"v1","addr":"172.21.15.149:6807","nonce":1064192336}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":1064192336},{"type":"v1","addr":"172.21.15.149:6805","nonce":1064192336}]},"public_addr":"172.21.15.149:6801/1064192336","cluster_addr":"172.21.15.149:6803/1064192336","heartbeat_back_addr":"172.21.15.149:6807/1064192336","heartbeat_front_addr":"172.21.15.149:6805/1064192336","state":["exists","up"]},{"osd":5,"uuid":"59adfbba-e1cf-461c-9698-6dd9755a808b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2676738694},{"type":"v1","addr":"172.21.15.149:6809","nonce":2676738694}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2676738694},{"type":"v1","addr":"172.21.15.149:6811","nonce":2676738694}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2676738694},{"type":"v1","addr":"172.21.15.149:6815","nonce":2676738694}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2676738694},{"type":"v1","addr":"172.21.15.149:6813","nonce":2676738694}]},"public_addr":"172.21.15.149:6809/2676738694","cluster_addr":"172.21.15.149:6811/2676738694","heartbeat_back_addr":"172.21.15.149:6815/2676738694","heartbeat_front_addr":"172.21.15.149:6813/2676738694","state":["exists","up"]},{"osd":6,"uuid":"265277c2-f8b1-46e4-b9b1-c6b37de45487","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":3434980700},{"type":"v1","addr":"172.21.15.149:6817","nonce":3434980700}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":3434980700},{"type":"v1","addr":"172.21.15.149:6819","nonce":3434980700}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":3434980700},{"type":"v1","addr":"172.21.15.149:6823","nonce":3434980700}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":3434980700},{"type":"v1","addr":"172.21.15.149:6821","nonce":3434980700}]},"public_addr":"172.21.15.149:6817/3434980700","cluster_addr":"172.21.15.149:6819/3434980700","heartbeat_back_addr":"172.21.15.149:6823/3434980700","heartbeat_front_addr":"172.21.15.149:6821/3434980700","state":["exists","up"]},{"osd":7,"uuid":"e49fc3e2-0333-4c4e-9edf-d85933fd46c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":1513400132},{"type":"v1","addr":"172.21.15.149:6825","nonce":1513400132}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":1513400132},{"type":"v1","addr":"172.21.15.149:6827","nonce":1513400132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":1513400132},{"type":"v1","addr":"172.21.15.149:6831","nonce":1513400132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":1513400132},{"type":"v1","addr":"172.21.15.149:6829","nonce":1513400132}]},"public_addr":"172.21.15.149:6825/1513400132","cluster_addr":"172.21.15.149:6827/1513400132","heartbeat_back_addr":"172.21.15.149:6831/1513400132","heartbeat_front_addr":"172.21.15.149:6829/1513400132","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:06:12.023674+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:06:45.135845+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:07:12.113109+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:07:38.761143+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:08:00.097037+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:08:23.879296+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:08:48.237571+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:09:12.503129+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.69:6800/2237930990":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/2087225558":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/2589858391":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/3578465763":"2024-04-23T23:04:13.845918+0000","172.21.15.69:6801/745479619":"2024-04-23T23:04:13.845918+0000","172.21.15.69:0/849543113":"2024-04-23T23:04:13.845918+0000","172.21.15.69:6800/745479619":"2024-04-23T23:04:13.845918+0000","172.21.15.69:0/3147401358":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/4144860677":"2024-04-23T23:03:44.182966+0000","172.21.15.69:0/2445874285":"2024-04-23T23:03:44.182966+0000","172.21.15.69:6801/2237930990":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/1041575105":"2024-04-23T23:04:13.845918+0000","172.21.15.69:0/493865482":"2024-04-23T23:03:44.182966+0000","172.21.15.69:6800/3986083499":"2024-04-23T23:03:44.182966+0000","172.21.15.69:6801/3986083499":"2024-04-23T23:03:44.182966+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T23:09:29.345 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:29 smithi069 ceph-mon[21859]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-04-22T23:09:29.345 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:29 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/499595387' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T23:09:29.370 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-22T23:09:29.370 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T23:09:29.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:29 smithi149 ceph-mon[23948]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-04-22T23:09:29.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:29 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/499595387' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T23:09:29.582 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:31.342 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:09:31.342 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":60,"fsid":"6928ed7e-00fc-11ef-bc93-c7b262605968","created":"2024-04-22T23:03:17.786186+0000","modified":"2024-04-22T23:09:17.463867+0000","last_up_change":"2024-04-22T23:09:13.448317+0000","last_in_change":"2024-04-22T23:08:57.123079+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T23:07:16.294432+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"7d98e167-3473-493e-8deb-1e10412bf5ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6802","nonce":2446576616},{"type":"v1","addr":"172.21.15.69:6803","nonce":2446576616}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6804","nonce":2446576616},{"type":"v1","addr":"172.21.15.69:6805","nonce":2446576616}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6808","nonce":2446576616},{"type":"v1","addr":"172.21.15.69:6809","nonce":2446576616}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6806","nonce":2446576616},{"type":"v1","addr":"172.21.15.69:6807","nonce":2446576616}]},"public_addr":"172.21.15.69:6803/2446576616","cluster_addr":"172.21.15.69:6805/2446576616","heartbeat_back_addr":"172.21.15.69:6809/2446576616","heartbeat_front_addr":"172.21.15.69:6807/2446576616","state":["exists","up"]},{"osd":1,"uuid":"9d15522c-2ca4-451c-a569-f31e25e5a08b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6810","nonce":2958036688},{"type":"v1","addr":"172.21.15.69:6811","nonce":2958036688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6812","nonce":2958036688},{"type":"v1","addr":"172.21.15.69:6813","nonce":2958036688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6816","nonce":2958036688},{"type":"v1","addr":"172.21.15.69:6817","nonce":2958036688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6814","nonce":2958036688},{"type":"v1","addr":"172.21.15.69:6815","nonce":2958036688}]},"public_addr":"172.21.15.69:6811/2958036688","cluster_addr":"172.21.15.69:6813/2958036688","heartbeat_back_addr":"172.21.15.69:6817/2958036688","heartbeat_front_addr":"172.21.15.69:6815/2958036688","state":["exists","up"]},{"osd":2,"uuid":"14d41cfc-5bca-40a5-bede-a02925fa4cb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6818","nonce":2823169129},{"type":"v1","addr":"172.21.15.69:6819","nonce":2823169129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6820","nonce":2823169129},{"type":"v1","addr":"172.21.15.69:6821","nonce":2823169129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6824","nonce":2823169129},{"type":"v1","addr":"172.21.15.69:6825","nonce":2823169129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6822","nonce":2823169129},{"type":"v1","addr":"172.21.15.69:6823","nonce":2823169129}]},"public_addr":"172.21.15.69:6819/2823169129","cluster_addr":"172.21.15.69:6821/2823169129","heartbeat_back_addr":"172.21.15.69:6825/2823169129","heartbeat_front_addr":"172.21.15.69:6823/2823169129","state":["exists","up"]},{"osd":3,"uuid":"40b632b4-3632-4281-bde9-e7aeadecf33f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6826","nonce":1031635705},{"type":"v1","addr":"172.21.15.69:6827","nonce":1031635705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6828","nonce":1031635705},{"type":"v1","addr":"172.21.15.69:6829","nonce":1031635705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6832","nonce":1031635705},{"type":"v1","addr":"172.21.15.69:6833","nonce":1031635705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6830","nonce":1031635705},{"type":"v1","addr":"172.21.15.69:6831","nonce":1031635705}]},"public_addr":"172.21.15.69:6827/1031635705","cluster_addr":"172.21.15.69:6829/1031635705","heartbeat_back_addr":"172.21.15.69:6833/1031635705","heartbeat_front_addr":"172.21.15.69:6831/1031635705","state":["exists","up"]},{"osd":4,"uuid":"67763500-c0be-4838-beb5-ea7f69c4395f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":1064192336},{"type":"v1","addr":"172.21.15.149:6801","nonce":1064192336}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":1064192336},{"type":"v1","addr":"172.21.15.149:6803","nonce":1064192336}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":1064192336},{"type":"v1","addr":"172.21.15.149:6807","nonce":1064192336}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":1064192336},{"type":"v1","addr":"172.21.15.149:6805","nonce":1064192336}]},"public_addr":"172.21.15.149:6801/1064192336","cluster_addr":"172.21.15.149:6803/1064192336","heartbeat_back_addr":"172.21.15.149:6807/1064192336","heartbeat_front_addr":"172.21.15.149:6805/1064192336","state":["exists","up"]},{"osd":5,"uuid":"59adfbba-e1cf-461c-9698-6dd9755a808b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2676738694},{"type":"v1","addr":"172.21.15.149:6809","nonce":2676738694}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2676738694},{"type":"v1","addr":"172.21.15.149:6811","nonce":2676738694}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2676738694},{"type":"v1","addr":"172.21.15.149:6815","nonce":2676738694}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2676738694},{"type":"v1","addr":"172.21.15.149:6813","nonce":2676738694}]},"public_addr":"172.21.15.149:6809/2676738694","cluster_addr":"172.21.15.149:6811/2676738694","heartbeat_back_addr":"172.21.15.149:6815/2676738694","heartbeat_front_addr":"172.21.15.149:6813/2676738694","state":["exists","up"]},{"osd":6,"uuid":"265277c2-f8b1-46e4-b9b1-c6b37de45487","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":3434980700},{"type":"v1","addr":"172.21.15.149:6817","nonce":3434980700}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":3434980700},{"type":"v1","addr":"172.21.15.149:6819","nonce":3434980700}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":3434980700},{"type":"v1","addr":"172.21.15.149:6823","nonce":3434980700}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":3434980700},{"type":"v1","addr":"172.21.15.149:6821","nonce":3434980700}]},"public_addr":"172.21.15.149:6817/3434980700","cluster_addr":"172.21.15.149:6819/3434980700","heartbeat_back_addr":"172.21.15.149:6823/3434980700","heartbeat_front_addr":"172.21.15.149:6821/3434980700","state":["exists","up"]},{"osd":7,"uuid":"e49fc3e2-0333-4c4e-9edf-d85933fd46c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":1513400132},{"type":"v1","addr":"172.21.15.149:6825","nonce":1513400132}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":1513400132},{"type":"v1","addr":"172.21.15.149:6827","nonce":1513400132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":1513400132},{"type":"v1","addr":"172.21.15.149:6831","nonce":1513400132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":1513400132},{"type":"v1","addr":"172.21.15.149:6829","nonce":1513400132}]},"public_addr":"172.21.15.149:6825/1513400132","cluster_addr":"172.21.15.149:6827/1513400132","heartbeat_back_addr":"172.21.15.149:6831/1513400132","heartbeat_front_addr":"172.21.15.149:6829/1513400132","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:06:12.023674+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:06:45.135845+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:07:12.113109+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:07:38.761143+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:08:00.097037+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:08:23.879296+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:08:48.237571+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T23:09:12.503129+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.69:6800/2237930990":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/2087225558":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/2589858391":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/3578465763":"2024-04-23T23:04:13.845918+0000","172.21.15.69:6801/745479619":"2024-04-23T23:04:13.845918+0000","172.21.15.69:0/849543113":"2024-04-23T23:04:13.845918+0000","172.21.15.69:6800/745479619":"2024-04-23T23:04:13.845918+0000","172.21.15.69:0/3147401358":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/4144860677":"2024-04-23T23:03:44.182966+0000","172.21.15.69:0/2445874285":"2024-04-23T23:03:44.182966+0000","172.21.15.69:6801/2237930990":"2024-04-23T23:05:07.221537+0000","172.21.15.69:0/1041575105":"2024-04-23T23:04:13.845918+0000","172.21.15.69:0/493865482":"2024-04-23T23:03:44.182966+0000","172.21.15.69:6800/3986083499":"2024-04-23T23:03:44.182966+0000","172.21.15.69:6801/3986083499":"2024-04-23T23:03:44.182966+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T23:09:31.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:31 smithi069 ceph-mon[21859]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-22T23:09:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:31 smithi149 ceph-mon[23948]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-22T23:09:32.033 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-22T23:09:32.034 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-22T23:09:32.034 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-22T23:09:32.035 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-22T23:09:32.035 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-22T23:09:32.035 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-22T23:09:32.036 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-22T23:09:32.036 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-22T23:09:32.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:32 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1514733390' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T23:09:32.435 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:32.441 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:32.442 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:32.443 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:32.445 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:32.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:32 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/1514733390' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T23:09:32.453 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:32.459 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:32.463 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:33.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:33 smithi069 ceph-mon[21859]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:33 smithi149 ceph-mon[23948]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:34.735 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:34 smithi069 ceph-mon[21859]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:34.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:34 smithi149 ceph-mon[23948]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:37.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:37 smithi069 ceph-mon[21859]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:37 smithi149 ceph-mon[23948]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:39.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:39 smithi069 ceph-mon[21859]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:39 smithi149 ceph-mon[23948]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:41.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:41 smithi069 ceph-mon[21859]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:41 smithi149 ceph-mon[23948]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:43.026 INFO:teuthology.orchestra.run.smithi069.stdout:171798691857 2024-04-22T23:09:43.026 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-22T23:09:43.330 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:43 smithi069 ceph-mon[21859]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:43 smithi149 ceph-mon[23948]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:43.759 INFO:teuthology.orchestra.run.smithi069.stdout:137438953495 2024-04-22T23:09:43.760 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-22T23:09:44.583 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:44 smithi069 ceph-mon[21859]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:44.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:44 smithi149 ceph-mon[23948]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:46.349 INFO:teuthology.orchestra.run.smithi069.stdout:60129542182 2024-04-22T23:09:46.349 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-22T23:09:47.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:47 smithi069 ceph-mon[21859]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:47 smithi149 ceph-mon[23948]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:47.933 INFO:teuthology.orchestra.run.smithi069.stdout:240518168585 2024-04-22T23:09:47.934 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-22T23:09:48.579 INFO:teuthology.orchestra.run.smithi069.stdout:81604378658 2024-04-22T23:09:48.580 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-22T23:09:49.039 INFO:teuthology.orchestra.run.smithi069.stdout:38654705709 2024-04-22T23:09:49.040 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-22T23:09:49.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:49 smithi069 ceph-mon[21859]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:49 smithi149 ceph-mon[23948]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:49.483 INFO:teuthology.orchestra.run.smithi069.stdout:206158430221 2024-04-22T23:09:49.484 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-22T23:09:49.564 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:49.564 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:49.565 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:49.566 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:49.566 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:49.979 INFO:teuthology.orchestra.run.smithi069.stdout:115964117020 2024-04-22T23:09:49.979 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-22T23:09:50.022 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:50.060 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:51.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:51 smithi069 ceph-mon[21859]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:51.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:51 smithi149 ceph-mon[23948]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:53.305 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:53 smithi069 ceph-mon[21859]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:53.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:53 smithi149 ceph-mon[23948]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:54.765 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:54 smithi069 ceph-mon[21859]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:54.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:54 smithi149 ceph-mon[23948]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:56.483 INFO:teuthology.orchestra.run.smithi069.stdout:60129542184 2024-04-22T23:09:57.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:57 smithi069 ceph-mon[21859]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:57.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:57 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2711956534' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T23:09:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:57 smithi149 ceph-mon[23948]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:57 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2711956534' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T23:09:57.550 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:09:58.309 INFO:teuthology.orchestra.run.smithi069.stdout:171798691861 2024-04-22T23:09:58.429 INFO:teuthology.orchestra.run.smithi069.stdout:206158430224 2024-04-22T23:09:58.693 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542182 got 60129542184 for osd.1 2024-04-22T23:09:58.693 DEBUG:teuthology.parallel:result is None 2024-04-22T23:09:59.287 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:59 smithi069 ceph-mon[21859]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:59.287 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:59 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1881055985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T23:09:59.288 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:09:59 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2637404823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T23:09:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:59 smithi149 ceph-mon[23948]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:09:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:59 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/1881055985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T23:09:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:09:59 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2637404823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T23:10:00.313 INFO:teuthology.orchestra.run.smithi069.stdout:38654705712 2024-04-22T23:10:00.313 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:00 smithi069 ceph-mon[21859]: overall HEALTH_OK 2024-04-22T23:10:00.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:00 smithi149 ceph-mon[23948]: overall HEALTH_OK 2024-04-22T23:10:00.544 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691857 got 171798691861 for osd.5 2024-04-22T23:10:00.545 DEBUG:teuthology.parallel:result is None 2024-04-22T23:10:01.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:01 smithi069 ceph-mon[21859]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:01.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:01 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2463116077' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T23:10:01.289 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705712 for osd.0 2024-04-22T23:10:01.289 DEBUG:teuthology.parallel:result is None 2024-04-22T23:10:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:01 smithi149 ceph-mon[23948]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:01 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2463116077' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T23:10:02.065 INFO:teuthology.orchestra.run.smithi069.stdout:240518168588 2024-04-22T23:10:02.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:02 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/270605212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T23:10:02.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:02 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/270605212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T23:10:02.470 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430221 got 206158430224 for osd.6 2024-04-22T23:10:02.470 DEBUG:teuthology.parallel:result is None 2024-04-22T23:10:03.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:03 smithi069 ceph-mon[21859]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:03 smithi149 ceph-mon[23948]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:03.735 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168585 got 240518168588 for osd.7 2024-04-22T23:10:03.735 DEBUG:teuthology.parallel:result is None 2024-04-22T23:10:04.113 INFO:teuthology.orchestra.run.smithi069.stdout:81604378661 2024-04-22T23:10:04.629 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:04 smithi069 ceph-mon[21859]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:04.630 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:04 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2373536296' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T23:10:04.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:04 smithi149 ceph-mon[23948]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:04.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:04 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2373536296' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T23:10:05.010 INFO:teuthology.orchestra.run.smithi069.stdout:137438953500 2024-04-22T23:10:05.629 INFO:teuthology.orchestra.run.smithi069.stdout:115964117023 2024-04-22T23:10:05.629 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:05 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/715763320' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T23:10:05.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:05 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/715763320' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T23:10:06.365 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117020 got 115964117023 for osd.3 2024-04-22T23:10:06.365 DEBUG:teuthology.parallel:result is None 2024-04-22T23:10:06.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:06 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/44788476' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T23:10:06.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:06 smithi069 ceph-mon[21859]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:06.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:06 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/44788476' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T23:10:06.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:06 smithi149 ceph-mon[23948]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:06.987 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378658 got 81604378661 for osd.2 2024-04-22T23:10:06.987 DEBUG:teuthology.parallel:result is None 2024-04-22T23:10:07.121 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953495 got 137438953500 for osd.4 2024-04-22T23:10:07.121 DEBUG:teuthology.parallel:result is None 2024-04-22T23:10:07.121 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-22T23:10:07.121 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-22T23:10:07.340 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:10:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:10:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:10:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:10:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:10:09.151 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:10:09.152 INFO:teuthology.orchestra.run.smithi069.stderr:dumped all 2024-04-22T23:10:09.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:09 smithi069 ceph-mon[21859]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:09 smithi149 ceph-mon[23948]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:09.823 INFO:teuthology.orchestra.run.smithi069.stdout:{"pg_ready":true,"pg_map":{"version":198,"stamp":"2024-04-22T23:10:08.037530+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48988,"kb_used_data":6364,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912228,"statfs":{"total":767960285184,"available":767910121472,"internally_reserved":0,"allocated":6516736,"data_stored":3570891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002194"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-22T23:09:17.478849+0000","last_change":"2024-04-22T23:09:17.478849+0000","last_active":"2024-04-22T23:09:17.478849+0000","last_peered":"2024-04-22T23:09:17.478849+0000","last_clean":"2024-04-22T23:09:17.478849+0000","last_became_active":"2024-04-22T23:09:17.478377+0000","last_became_peered":"2024-04-22T23:09:17.478377+0000","last_unstale":"2024-04-22T23:09:17.478849+0000","last_undegraded":"2024-04-22T23:09:17.478849+0000","last_fullsized":"2024-04-22T23:09:17.478849+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-22T23:07:16.428547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-22T23:07:16.428547+0000","last_clean_scrub_stamp":"2024-04-22T23:07:16.428547+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-24T05:38:49.827179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2662400,"data_stored":2644640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168589,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":776607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.745}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69699999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70899999999999996}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.755}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65000000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61699999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59099999999999997}]}]},{"osd":6,"up_from":48,"seq":206158430226,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6180,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738972,"statfs":{"total":95995035648,"available":95988707328,"internally_reserved":0,"allocated":1142784,"data_stored":776607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 23:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.54800000000000004}]},{"osd":1,"last update":"Mon Apr 22 23:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.52100000000000002}]},{"osd":2,"last update":"Mon Apr 22 23:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.51400000000000001}]},{"osd":3,"last update":"Mon Apr 22 23:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.501},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":8.2420000000000009,"5min":8.2420000000000009,"15min":8.2420000000000009},"last":0.48099999999999998}]},{"osd":4,"last update":"Mon Apr 22 23:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.47199999999999998}]},{"osd":5,"last update":"Mon Apr 22 23:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.40000000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44400000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 23:09:53 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.35699999999999998,"15min":0.34300000000000003},"min":{"1min":0.26700000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.52200000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.35899999999999999,"15min":0.34300000000000003},"min":{"1min":0.23999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.53500000000000003,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.34899999999999998}]},{"osd":2,"last update":"Mon Apr 22 23:09:19 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.38600000000000001,"15min":0.38100000000000001},"min":{"1min":0.31,"5min":0.253,"15min":0.253},"max":{"1min":0.50800000000000001,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.311},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.375,"15min":0.36699999999999999},"min":{"1min":0.308,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.32600000000000001}]},{"osd":3,"last update":"Mon Apr 22 23:09:44 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.437,"15min":0.442},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55000000000000004,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.46300000000000002,"15min":0.46500000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.56699999999999995,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Apr 22 23:09:05 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.625}]},{"osd":5,"last update":"Mon Apr 22 23:09:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.66800000000000004}]},{"osd":6,"last update":"Mon Apr 22 23:09:53 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":16.513999999999999,"5min":16.513999999999999,"15min":16.513999999999999},"last":0.57399999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61699999999999999}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Apr 22 23:09:57 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.34200000000000003,"15min":0.32400000000000001},"min":{"1min":0.23200000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.55400000000000005,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.35699999999999998,"15min":0.33600000000000002},"min":{"1min":0.28699999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.32000000000000001}]},{"osd":2,"last update":"Mon Apr 22 23:09:21 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.39300000000000002,"15min":0.38100000000000001},"min":{"1min":0.318,"5min":0.246,"15min":0.246},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.437,"5min":0.40600000000000003,"15min":0.40100000000000002},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.378}]},{"osd":3,"last update":"Mon Apr 22 23:09:43 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.42399999999999999,"15min":0.42499999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.45300000000000001,"15min":0.45400000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.36399999999999999}]},{"osd":4,"last update":"Mon Apr 22 23:09:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.61699999999999999}]},{"osd":5,"last update":"Mon Apr 22 23:09:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.66500000000000004}]},{"osd":6,"last update":"Mon Apr 22 23:09:51 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.68999999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70799999999999996}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 23:09:17 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.35199999999999998,"15min":0.34000000000000002},"min":{"1min":0.35399999999999998,"5min":0.249,"15min":0.249},"max":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.374,"15min":0.36699999999999999},"min":{"1min":0.30399999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.28599999999999998}]},{"osd":1,"last update":"Mon Apr 22 23:09:17 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.373,"15min":0.371},"min":{"1min":0.26300000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.38700000000000001,"15min":0.375},"min":{"1min":0.33000000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.42099999999999999}]},{"osd":3,"last update":"Mon Apr 22 23:09:45 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.46000000000000002,"15min":0.46600000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.55000000000000004,"5min":0.629,"15min":0.629},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.44500000000000001,"15min":0.45300000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.52700000000000002,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.435}]},{"osd":4,"last update":"Mon Apr 22 23:10:04 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.53100000000000003,"15min":0.53600000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.65100000000000002,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57699999999999996,"15min":0.57899999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.70799999999999996,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.56200000000000006}]},{"osd":5,"last update":"Mon Apr 22 23:09:28 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.61899999999999999}]},{"osd":6,"last update":"Mon Apr 22 23:09:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.57299999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61399999999999999}]}]},{"osd":3,"up_from":27,"seq":115964117024,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 23:09:48 2024","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.36199999999999999,"15min":0.36399999999999999},"min":{"1min":0.24399999999999999,"5min":0.215,"15min":0.215},"max":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.44400000000000001}]},{"osd":1,"last update":"Mon Apr 22 23:09:48 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.41799999999999998,"15min":0.42199999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.38,"5min":0.40400000000000003,"15min":0.40799999999999997},"min":{"1min":0.29099999999999998,"5min":0.248,"15min":0.248},"max":{"1min":0.51600000000000001,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.496}]},{"osd":2,"last update":"Mon Apr 22 23:09:48 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.45200000000000001,"15min":0.45600000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.54900000000000004,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.437,"15min":0.443},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.54800000000000004}]},{"osd":4,"last update":"Mon Apr 22 23:10:02 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.58699999999999997,"15min":0.57999999999999996},"min":{"1min":0.41299999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.57599999999999996,"15min":0.57199999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.80500000000000005}]},{"osd":5,"last update":"Mon Apr 22 23:09:31 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.879}]},{"osd":6,"last update":"Mon Apr 22 23:09:51 2024","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":19.219000000000001,"5min":19.219000000000001,"15min":19.219000000000001},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.84999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76400000000000001}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 23:09:05 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.66100000000000003}]},{"osd":1,"last update":"Mon Apr 22 23:09:05 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.58199999999999996}]},{"osd":2,"last update":"Mon Apr 22 23:09:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.60899999999999999}]},{"osd":3,"last update":"Mon Apr 22 23:09:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.63500000000000001}]},{"osd":5,"last update":"Mon Apr 22 23:09:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.40899999999999997}]},{"osd":6,"last update":"Mon Apr 22 23:09:51 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.42299999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44600000000000001}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":247679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 23:09:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.46400000000000002}]},{"osd":1,"last update":"Mon Apr 22 23:09:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.52400000000000002}]},{"osd":2,"last update":"Mon Apr 22 23:09:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.55700000000000005}]},{"osd":3,"last update":"Mon Apr 22 23:09:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.58299999999999996}]},{"osd":4,"last update":"Mon Apr 22 23:09:25 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.379},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Apr 22 23:09:51 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.625}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-22T23:10:09.825 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-22T23:10:10.034 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:10:10.392 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:10 smithi069 ceph-mon[21859]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:10:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:10 smithi149 ceph-mon[23948]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:10:11.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:11 smithi069 ceph-mon[21859]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:11 smithi149 ceph-mon[23948]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:11.837 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:10:11.838 INFO:teuthology.orchestra.run.smithi069.stderr:dumped all 2024-04-22T23:10:12.451 INFO:teuthology.orchestra.run.smithi069.stdout:{"pg_ready":true,"pg_map":{"version":199,"stamp":"2024-04-22T23:10:10.037780+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48988,"kb_used_data":6364,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912228,"statfs":{"total":767960285184,"available":767910121472,"internally_reserved":0,"allocated":6516736,"data_stored":3570891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001982"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-22T23:09:17.478849+0000","last_change":"2024-04-22T23:09:17.478849+0000","last_active":"2024-04-22T23:09:17.478849+0000","last_peered":"2024-04-22T23:09:17.478849+0000","last_clean":"2024-04-22T23:09:17.478849+0000","last_became_active":"2024-04-22T23:09:17.478377+0000","last_became_peered":"2024-04-22T23:09:17.478377+0000","last_unstale":"2024-04-22T23:09:17.478849+0000","last_undegraded":"2024-04-22T23:09:17.478849+0000","last_fullsized":"2024-04-22T23:09:17.478849+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-22T23:07:16.428547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-22T23:07:16.428547+0000","last_clean_scrub_stamp":"2024-04-22T23:07:16.428547+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-24T05:38:49.827179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2662400,"data_stored":2644640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":776607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.503}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54700000000000004}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.497}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47699999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42199999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57199999999999995}]}]},{"osd":6,"up_from":48,"seq":206158430226,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6180,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738972,"statfs":{"total":95995035648,"available":95988707328,"internally_reserved":0,"allocated":1142784,"data_stored":776607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 23:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.54800000000000004}]},{"osd":1,"last update":"Mon Apr 22 23:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.52100000000000002}]},{"osd":2,"last update":"Mon Apr 22 23:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.51400000000000001}]},{"osd":3,"last update":"Mon Apr 22 23:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.501},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":8.2420000000000009,"5min":8.2420000000000009,"15min":8.2420000000000009},"last":0.48099999999999998}]},{"osd":4,"last update":"Mon Apr 22 23:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.47199999999999998}]},{"osd":5,"last update":"Mon Apr 22 23:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.40000000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44400000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 23:09:53 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.35699999999999998,"15min":0.34300000000000003},"min":{"1min":0.26700000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.52200000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.35899999999999999,"15min":0.34300000000000003},"min":{"1min":0.23999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.53500000000000003,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.34899999999999998}]},{"osd":2,"last update":"Mon Apr 22 23:09:19 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.38600000000000001,"15min":0.38100000000000001},"min":{"1min":0.31,"5min":0.253,"15min":0.253},"max":{"1min":0.50800000000000001,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.311},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.375,"15min":0.36699999999999999},"min":{"1min":0.308,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.32600000000000001}]},{"osd":3,"last update":"Mon Apr 22 23:09:44 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.437,"15min":0.442},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55000000000000004,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.46300000000000002,"15min":0.46500000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.56699999999999995,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Apr 22 23:09:05 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.625}]},{"osd":5,"last update":"Mon Apr 22 23:09:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.66800000000000004}]},{"osd":6,"last update":"Mon Apr 22 23:09:53 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":16.513999999999999,"5min":16.513999999999999,"15min":16.513999999999999},"last":0.57399999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61699999999999999}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Apr 22 23:09:57 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.34200000000000003,"15min":0.32400000000000001},"min":{"1min":0.23200000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.55400000000000005,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.35699999999999998,"15min":0.33600000000000002},"min":{"1min":0.28699999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.34599999999999997}]},{"osd":2,"last update":"Mon Apr 22 23:09:21 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.39300000000000002,"15min":0.38100000000000001},"min":{"1min":0.318,"5min":0.246,"15min":0.246},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.437,"5min":0.40600000000000003,"15min":0.40100000000000002},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.33300000000000002}]},{"osd":3,"last update":"Mon Apr 22 23:09:43 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.42399999999999999,"15min":0.42499999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.308},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.45300000000000001,"15min":0.45400000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.35799999999999998}]},{"osd":4,"last update":"Mon Apr 22 23:10:07 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.45700000000000002,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.72699999999999998,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.59699999999999998,"15min":0.58799999999999997},"min":{"1min":0.442,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.83699999999999997,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.58399999999999996}]},{"osd":5,"last update":"Mon Apr 22 23:09:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.57699999999999996}]},{"osd":6,"last update":"Mon Apr 22 23:09:51 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.61299999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52300000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 23:09:17 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.35199999999999998,"15min":0.34000000000000002},"min":{"1min":0.35399999999999998,"5min":0.249,"15min":0.249},"max":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.374,"15min":0.36699999999999999},"min":{"1min":0.30399999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.28599999999999998}]},{"osd":1,"last update":"Mon Apr 22 23:09:17 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.373,"15min":0.371},"min":{"1min":0.26300000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.38700000000000001,"15min":0.375},"min":{"1min":0.33000000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.42099999999999999}]},{"osd":3,"last update":"Mon Apr 22 23:09:45 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.46000000000000002,"15min":0.46600000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.55000000000000004,"5min":0.629,"15min":0.629},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.44500000000000001,"15min":0.45300000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.52700000000000002,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.435}]},{"osd":4,"last update":"Mon Apr 22 23:10:04 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.53100000000000003,"15min":0.53600000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.65100000000000002,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57699999999999996,"15min":0.57899999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.70799999999999996,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.56200000000000006}]},{"osd":5,"last update":"Mon Apr 22 23:09:28 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.61899999999999999}]},{"osd":6,"last update":"Mon Apr 22 23:09:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.57299999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61399999999999999}]}]},{"osd":3,"up_from":27,"seq":115964117024,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 23:09:48 2024","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.36199999999999999,"15min":0.36399999999999999},"min":{"1min":0.24399999999999999,"5min":0.215,"15min":0.215},"max":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.44400000000000001}]},{"osd":1,"last update":"Mon Apr 22 23:09:48 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.41799999999999998,"15min":0.42199999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.38,"5min":0.40400000000000003,"15min":0.40799999999999997},"min":{"1min":0.29099999999999998,"5min":0.248,"15min":0.248},"max":{"1min":0.51600000000000001,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.496}]},{"osd":2,"last update":"Mon Apr 22 23:09:48 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.45200000000000001,"15min":0.45600000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.54900000000000004,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.437,"15min":0.443},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.54800000000000004}]},{"osd":4,"last update":"Mon Apr 22 23:10:02 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.58699999999999997,"15min":0.57999999999999996},"min":{"1min":0.41299999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.57599999999999996,"15min":0.57199999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.80500000000000005}]},{"osd":5,"last update":"Mon Apr 22 23:09:31 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.879}]},{"osd":6,"last update":"Mon Apr 22 23:09:51 2024","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":19.219000000000001,"5min":19.219000000000001,"15min":19.219000000000001},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.84999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76400000000000001}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 23:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55900000000000005,"15min":0.56200000000000006},"min":{"1min":0.379,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69499999999999995,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58299999999999996,"15min":0.57899999999999996},"min":{"1min":0.442,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.73099999999999998,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.70699999999999996}]},{"osd":1,"last update":"Mon Apr 22 23:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.36799999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.626},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55600000000000005,"15min":0.55800000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.747,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.72399999999999998}]},{"osd":2,"last update":"Mon Apr 22 23:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.54000000000000004,"15min":0.53700000000000003},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53000000000000003,"15min":0.52900000000000003},"min":{"1min":0.41699999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.753,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.56299999999999994}]},{"osd":3,"last update":"Mon Apr 22 23:10:06 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.53800000000000003,"15min":0.53400000000000003},"min":{"1min":0.371,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72099999999999997,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.39700000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77800000000000002,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.58199999999999996}]},{"osd":5,"last update":"Mon Apr 22 23:09:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.69199999999999995}]},{"osd":6,"last update":"Mon Apr 22 23:09:51 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.61199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66000000000000003}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":247679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 23:09:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.46400000000000002}]},{"osd":1,"last update":"Mon Apr 22 23:09:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.52400000000000002}]},{"osd":2,"last update":"Mon Apr 22 23:09:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.55700000000000005}]},{"osd":3,"last update":"Mon Apr 22 23:09:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.58299999999999996}]},{"osd":4,"last update":"Mon Apr 22 23:09:25 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.379},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Apr 22 23:09:51 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.625}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-22T23:10:12.452 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-22T23:10:12.452 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-22T23:10:12.453 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-22T23:10:12.453 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-22T23:10:12.661 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/mon.smithi069/config 2024-04-22T23:10:13.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:13 smithi069 ceph-mon[21859]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:10:13.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:13 smithi069 ceph-mon[21859]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:13 smithi149 ceph-mon[23948]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:10:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:13 smithi149 ceph-mon[23948]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:14.446 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:10:14.446 INFO:teuthology.orchestra.run.smithi069.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-22T23:10:14.799 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:14 smithi069 ceph-mon[21859]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:14.800 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:14 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1686259988' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T23:10:14.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:14 smithi149 ceph-mon[23948]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:14.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:14 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/1686259988' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T23:10:15.105 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-22T23:10:15.105 INFO:tasks.cephadm:Setup complete, yielding 2024-04-22T23:10:15.106 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T23:10:15.116 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T23:10:15.116 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'ceph orch status' 2024-04-22T23:10:16.289 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:10:16.290 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:16.290 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:16 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:10:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:10:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:16.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:16 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:10:16.990 INFO:teuthology.orchestra.run.smithi069.stdout:Backend: cephadm 2024-04-22T23:10:16.990 INFO:teuthology.orchestra.run.smithi069.stdout:Available: Yes 2024-04-22T23:10:16.990 INFO:teuthology.orchestra.run.smithi069.stdout:Paused: No 2024-04-22T23:10:17.288 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:17 smithi069 ceph-mon[21859]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:17.288 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:17 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:17.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:17 smithi149 ceph-mon[23948]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:17.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:17 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:17.604 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'ceph orch ps' 2024-04-22T23:10:18.161 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:18 smithi069 ceph-mon[21859]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:18.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:18 smithi149 ceph-mon[23948]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:19.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:19 smithi069 ceph-mon[21859]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:19.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:19 smithi149 ceph-mon[23948]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:19.670 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:10:19.670 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (4m) 2m ago 5m 14.6M - ba2b418f427c e0aa152f63bd 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (5m) 2m ago 5m 6400k - 17.2.0 e1d6a67b021e f29a69890fd2 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (5m) 66s ago 5m 6400k - 17.2.0 e1d6a67b021e d3576856bdfd 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (4m) 2m ago 5m 42.5M - 8.3.5 dad864ee21e9 c5880e24c661 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:9283 running (6m) 2m ago 6m 447M - 17.2.0 e1d6a67b021e 98c4390893c8 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283 running (4m) 66s ago 4m 412M - 17.2.0 e1d6a67b021e fa2d6964da01 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (6m) 2m ago 7m 42.0M 2048M 17.2.0 e1d6a67b021e 008ad8fe38c6 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (4m) 66s ago 4m 39.7M 2048M 17.2.0 e1d6a67b021e 1a87df28e650 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (5m) 2m ago 5m 16.6M - 1dbe0e931976 1175a40c25d7 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (4m) 66s ago 4m 16.6M - 1dbe0e931976 fd4ddbc4e0f2 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (4m) 2m ago 4m 35.5M 3216M 17.2.0 e1d6a67b021e 777db7a3d809 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (3m) 2m ago 3m 34.5M 3216M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (3m) 2m ago 3m 34.2M 3216M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:10:19.671 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (2m) 2m ago 2m 13.1M 3216M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:10:19.672 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (2m) 66s ago 2m 33.3M 1166M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:10:19.672 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (2m) 66s ago 119s 35.8M 1166M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:10:19.672 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (95s) 66s ago 95s 31.0M 1166M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:10:19.672 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (71s) 66s ago 71s 10.9M 1166M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:10:19.672 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (4m) 2m ago 4m 53.9M - 514e6a882f6e b8ea50a9c418 2024-04-22T23:10:20.333 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'ceph orch ls' 2024-04-22T23:10:21.291 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:21 smithi069 ceph-mon[21859]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:21.291 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:21 smithi069 ceph-mon[21859]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:21 smithi149 ceph-mon[23948]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:21 smithi149 ceph-mon[23948]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:22.239 INFO:teuthology.orchestra.run.smithi069.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-22T23:10:22.239 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-04-22T23:10:22.239 INFO:teuthology.orchestra.run.smithi069.stdout:crash 2/2 2m ago 6m * 2024-04-22T23:10:22.239 INFO:teuthology.orchestra.run.smithi069.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-04-22T23:10:22.239 INFO:teuthology.orchestra.run.smithi069.stdout:mgr 2/2 2m ago 6m count:2 2024-04-22T23:10:22.239 INFO:teuthology.orchestra.run.smithi069.stdout:mon 2/2 2m ago 5m smithi069:172.21.15.69=smithi069;smithi149:172.21.15.149=smithi149;count:2 2024-04-22T23:10:22.240 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-04-22T23:10:22.240 INFO:teuthology.orchestra.run.smithi069.stdout:osd 8 2m ago - 2024-04-22T23:10:22.240 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-04-22T23:10:22.882 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'ceph orch host ls' 2024-04-22T23:10:23.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:23 smithi069 ceph-mon[21859]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:23.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:23 smithi069 ceph-mon[21859]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:23.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:23 smithi149 ceph-mon[23948]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:23.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:23 smithi149 ceph-mon[23948]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:24.858 INFO:teuthology.orchestra.run.smithi069.stdout:HOST ADDR LABELS STATUS 2024-04-22T23:10:24.858 INFO:teuthology.orchestra.run.smithi069.stdout:smithi069 172.21.15.69 2024-04-22T23:10:24.858 INFO:teuthology.orchestra.run.smithi069.stdout:smithi149 172.21.15.149 2024-04-22T23:10:24.858 INFO:teuthology.orchestra.run.smithi069.stdout:2 hosts in cluster 2024-04-22T23:10:24.859 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:24 smithi069 ceph-mon[21859]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:24.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:24 smithi149 ceph-mon[23948]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:25.493 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'ceph orch device ls' 2024-04-22T23:10:25.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:25 smithi069 ceph-mon[21859]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:25 smithi149 ceph-mon[23948]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:26.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:26 smithi149 ceph-mon[23948]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:26.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:26 smithi069 ceph-mon[21859]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:27.380 INFO:teuthology.orchestra.run.smithi069.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-04-22T23:10:27.380 INFO:teuthology.orchestra.run.smithi069.stdout:smithi069 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400Y0400BGN 400G LVM detected, locked 2024-04-22T23:10:27.380 INFO:teuthology.orchestra.run.smithi069.stdout:smithi149 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT623300CK400BGN 400G LVM detected, locked 2024-04-22T23:10:28.125 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-22T23:10:28.137 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T23:10:28.138 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-22T23:10:28.164 INFO:teuthology.orchestra.run.smithi069.stderr:+ systemctl stop nfs-server 2024-04-22T23:10:28.171 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi149.front.sepia.ceph.com 2024-04-22T23:10:28.171 DEBUG:teuthology.orchestra.run.smithi149:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-22T23:10:28.201 INFO:teuthology.orchestra.run.smithi149.stderr:+ systemctl stop nfs-server 2024-04-22T23:10:28.210 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T23:10:28.218 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T23:10:28.218 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-04-22T23:10:29.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:29 smithi069 ceph-mon[21859]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:29.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:29 smithi069 ceph-mon[21859]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:29.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:29 smithi149 ceph-mon[23948]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:29.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:29 smithi149 ceph-mon[23948]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:31.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:31 smithi069 ceph-mon[21859]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:31.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:31 smithi069 ceph-mon[21859]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:31.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:31 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-22T23:10:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:31 smithi149 ceph-mon[23948]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:31 smithi149 ceph-mon[23948]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:31 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-22T23:10:32.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:32 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-22T23:10:32.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:32 smithi069 ceph-mon[21859]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T23:10:32.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:32 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-22T23:10:32.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:32 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069[21855]: 2024-04-22T23:10:32.112+0000 7f37c9019700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T23:10:32.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:32 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-22T23:10:32.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:32 smithi149 ceph-mon[23948]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T23:10:32.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:32 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-22T23:10:32.862 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-04-22T23:10:32.871 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-04-22T23:10:32.871 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-22T23:10:33.375 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: pgmap v211: 33 pgs: 8 creating+peering, 1 active+clean, 24 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:33.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-22T23:10:33.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T23:10:33.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-22T23:10:33.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T23:10:33.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-22T23:10:33.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T23:10:33.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-22T23:10:33.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: fsmap foofs:0 2024-04-22T23:10:33.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: Saving service mds.foofs spec with placement count:2 2024-04-22T23:10:33.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:33.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:10:33.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:33.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:10:33.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:33.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:33.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.jiyres", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T23:10:33.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.jiyres", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-22T23:10:33.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:33.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:33 smithi069 ceph-mon[21859]: Deploying daemon mds.foofs.smithi069.jiyres on smithi069 2024-04-22T23:10:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: pgmap v211: 33 pgs: 8 creating+peering, 1 active+clean, 24 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: fsmap foofs:0 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: Saving service mds.foofs spec with placement count:2 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:33.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.jiyres", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T23:10:33.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.jiyres", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-22T23:10:33.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:33.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:33 smithi149 ceph-mon[23948]: Deploying daemon mds.foofs.smithi069.jiyres on smithi069 2024-04-22T23:10:34.373 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:34 smithi069 ceph-mon[21859]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T23:10:34.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:34 smithi149 ceph-mon[23948]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T23:10:35.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:35 smithi069 ceph-mon[21859]: pgmap v215: 65 pgs: 12 creating+peering, 11 active+clean, 42 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:35.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:35 smithi069 ceph-mon[21859]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T23:10:35.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:35 smithi149 ceph-mon[23948]: pgmap v215: 65 pgs: 12 creating+peering, 11 active+clean, 42 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:35.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:35 smithi149 ceph-mon[23948]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T23:10:36.159 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:10:36.159 INFO:teuthology.orchestra.run.smithi069.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-22T23:04:02.398445Z", "last_refresh": "2024-04-22T23:07:41.609871Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-22T23:03:57.283881Z", "last_refresh": "2024-04-22T23:07:41.609932Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-22T23:03:59.884436Z", "last_refresh": "2024-04-22T23:07:41.609992Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-22T23:10:32.131901Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-22T23:10:32.123823Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-22T23:03:55.163282Z", "last_refresh": "2024-04-22T23:07:41.609802Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi069:172.21.15.69=smithi069", "smithi149:172.21.15.149=smithi149"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-22T23:04:47.223932Z", "last_refresh": "2024-04-22T23:07:41.609664Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-22T23:04:01.114168Z", "last_refresh": "2024-04-22T23:07:41.610050Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-22T23:07:41.610166Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-22T23:03:58.572865Z", "last_refresh": "2024-04-22T23:07:41.610108Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-22T23:10:36.888 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-22T23:10:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: pgmap v217: 65 pgs: 12 creating+peering, 45 active+clean, 8 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.ickhha", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.ickhha", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: Deploying daemon mds.foofs.smithi149.ickhha on smithi149 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: mds.? [v2:172.21.15.69:6834/1879302209,v1:172.21.15.69:6835/1879302209] up:boot 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: daemon mds.foofs.smithi069.jiyres assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: Cluster is now healthy 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: fsmap foofs:0 1 up:standby 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:10:37.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:creating} 2024-04-22T23:10:37.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:37 smithi149 ceph-mon[23948]: daemon mds.foofs.smithi069.jiyres is now active in filesystem foofs as rank 0 2024-04-22T23:10:37.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: pgmap v217: 65 pgs: 12 creating+peering, 45 active+clean, 8 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:10:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.ickhha", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T23:10:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.ickhha", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-22T23:10:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: Deploying daemon mds.foofs.smithi149.ickhha on smithi149 2024-04-22T23:10:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: mds.? [v2:172.21.15.69:6834/1879302209,v1:172.21.15.69:6835/1879302209] up:boot 2024-04-22T23:10:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: daemon mds.foofs.smithi069.jiyres assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-22T23:10:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-22T23:10:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-22T23:10:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: Cluster is now healthy 2024-04-22T23:10:37.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: fsmap foofs:0 1 up:standby 2024-04-22T23:10:37.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:10:37.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:creating} 2024-04-22T23:10:37.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:37 smithi069 ceph-mon[21859]: daemon mds.foofs.smithi069.jiyres is now active in filesystem foofs as rank 0 2024-04-22T23:10:37.889 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-22T23:10:39.305 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:39 smithi069 ceph-mon[21859]: pgmap v218: 65 pgs: 4 creating+peering, 61 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 344 B/s wr, 0 op/s 2024-04-22T23:10:39.306 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:39 smithi069 ceph-mon[21859]: mds.? [v2:172.21.15.69:6834/1879302209,v1:172.21.15.69:6835/1879302209] up:active 2024-04-22T23:10:39.306 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:39 smithi069 ceph-mon[21859]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 2024-04-22T23:10:39.306 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:39 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:39.306 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:39 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:10:39.306 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:39 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:39.306 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:39 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:10:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:39 smithi149 ceph-mon[23948]: pgmap v218: 65 pgs: 4 creating+peering, 61 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 344 B/s wr, 0 op/s 2024-04-22T23:10:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:39 smithi149 ceph-mon[23948]: mds.? [v2:172.21.15.69:6834/1879302209,v1:172.21.15.69:6835/1879302209] up:active 2024-04-22T23:10:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:39 smithi149 ceph-mon[23948]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 2024-04-22T23:10:39.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:39 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:39.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:39 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:10:39.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:39 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:39.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:39 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:10:39.970 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:10:39.970 INFO:teuthology.orchestra.run.smithi069.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-22T23:04:02.398445Z", "last_refresh": "2024-04-22T23:07:41.609871Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-22T23:03:57.283881Z", "last_refresh": "2024-04-22T23:07:41.609932Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-22T23:03:59.884436Z", "last_refresh": "2024-04-22T23:07:41.609992Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-22T23:10:32.131901Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-22T23:10:32.123823Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-22T23:03:55.163282Z", "last_refresh": "2024-04-22T23:07:41.609802Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi069:172.21.15.69=smithi069", "smithi149:172.21.15.149=smithi149"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-22T23:04:47.223932Z", "last_refresh": "2024-04-22T23:07:41.609664Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-22T23:04:01.114168Z", "last_refresh": "2024-04-22T23:07:41.610050Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-22T23:07:41.610166Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-22T23:03:58.572865Z", "last_refresh": "2024-04-22T23:07:41.610108Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-22T23:10:40.750 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:40 smithi149 ceph-mon[23948]: mds.? [v2:172.21.15.149:6832/1034958666,v1:172.21.15.149:6833/1034958666] up:boot 2024-04-22T23:10:40.750 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:40 smithi149 ceph-mon[23948]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 1 up:standby 2024-04-22T23:10:40.751 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:40 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:10:40.751 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:40 smithi149 ceph-mon[23948]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 1 up:standby 2024-04-22T23:10:40.751 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:40 smithi149 ceph-mon[23948]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:10:40.751 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:40 smithi149 ceph-mon[23948]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 2 op/s 2024-04-22T23:10:40.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:40 smithi069 ceph-mon[21859]: mds.? [v2:172.21.15.149:6832/1034958666,v1:172.21.15.149:6833/1034958666] up:boot 2024-04-22T23:10:40.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:40 smithi069 ceph-mon[21859]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 1 up:standby 2024-04-22T23:10:40.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:40 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:10:40.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:40 smithi069 ceph-mon[21859]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 1 up:standby 2024-04-22T23:10:40.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:40 smithi069 ceph-mon[21859]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:10:40.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:40 smithi069 ceph-mon[21859]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 2 op/s 2024-04-22T23:10:41.269 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-22T23:10:41.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:41 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:42.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:41 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:42.269 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-22T23:10:43.025 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:42 smithi069 ceph-mon[21859]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-04-22T23:10:43.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:42 smithi149 ceph-mon[23948]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-04-22T23:10:44.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:44 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:44 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:45.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:45 smithi069 ceph-mon[21859]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-22T23:10:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:45 smithi149 ceph-mon[23948]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-22T23:10:45.591 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:10:45.591 INFO:teuthology.orchestra.run.smithi069.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-22T23:04:02.398445Z", "last_refresh": "2024-04-22T23:07:41.609871Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-22T23:03:57.283881Z", "last_refresh": "2024-04-22T23:07:41.609932Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-22T23:03:59.884436Z", "last_refresh": "2024-04-22T23:07:41.609992Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-22T23:10:32.131901Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-22T23:10:32.123823Z", "last_refresh": "2024-04-22T23:10:40.754230Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-22T23:03:55.163282Z", "last_refresh": "2024-04-22T23:07:41.609802Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi069:172.21.15.69=smithi069", "smithi149:172.21.15.149=smithi149"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-22T23:04:47.223932Z", "last_refresh": "2024-04-22T23:07:41.609664Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-22T23:04:01.114168Z", "last_refresh": "2024-04-22T23:07:41.610050Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-22T23:07:41.610166Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-22T23:03:58.572865Z", "last_refresh": "2024-04-22T23:07:41.610108Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-22T23:10:46.391 INFO:tasks.cephadm:mds.foofs has 1/2 2024-04-22T23:10:47.392 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-22T23:10:47.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:47 smithi069 ceph-mon[21859]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:10:47.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:47 smithi069 ceph-mon[21859]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-22T23:10:47.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:47 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:47.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:47 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:47.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:47 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:10:47.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:47 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:47.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:47 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:10:47.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:47 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:47 smithi149 ceph-mon[23948]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:10:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:47 smithi149 ceph-mon[23948]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-22T23:10:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:47 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:47 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:47 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:10:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:47 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:47 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:10:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:47 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:49.291 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:10:49.292 INFO:teuthology.orchestra.run.smithi069.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-22T23:04:02.398445Z", "last_refresh": "2024-04-22T23:10:46.663383Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-22T23:03:57.283881Z", "last_refresh": "2024-04-22T23:10:40.753636Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-22T23:03:59.884436Z", "last_refresh": "2024-04-22T23:10:46.663510Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-22T23:10:32.131901Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-22T23:10:32.123823Z", "last_refresh": "2024-04-22T23:10:40.754230Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-22T23:03:55.163282Z", "last_refresh": "2024-04-22T23:10:40.753784Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi069:172.21.15.69=smithi069", "smithi149:172.21.15.149=smithi149"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-22T23:04:47.223932Z", "last_refresh": "2024-04-22T23:10:40.753857Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-22T23:04:01.114168Z", "last_refresh": "2024-04-22T23:10:40.753923Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-22T23:10:40.753987Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-22T23:03:58.572865Z", "last_refresh": "2024-04-22T23:10:46.663646Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-22T23:10:49.292 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:49 smithi069 ceph-mon[21859]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-22T23:10:49.292 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:49 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:49 smithi149 ceph-mon[23948]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-22T23:10:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:49 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:49.879 INFO:tasks.cephadm:mds.foofs has 2/2 2024-04-22T23:10:49.880 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T23:10:49.892 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T23:10:49.892 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-04-22T23:10:50.319 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:50 smithi069 ceph-mon[21859]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:10:50.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:50 smithi149 ceph-mon[23948]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:10:51.220 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:51 smithi069 ceph-mon[21859]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 938 B/s wr, 3 op/s 2024-04-22T23:10:51.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:51 smithi149 ceph-mon[23948]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 938 B/s wr, 3 op/s 2024-04-22T23:10:52.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:52 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-22T23:10:52.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:52 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-22T23:10:53.252 INFO:teuthology.orchestra.run.smithi069.stdout:NFS Cluster Created Successfully 2024-04-22T23:10:53.588 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:53 smithi069 ceph-mon[21859]: from='client.14516 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:53.588 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:53 smithi069 ceph-mon[21859]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 2 op/s 2024-04-22T23:10:53.588 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:53 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-22T23:10:53.588 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:53 smithi069 ceph-mon[21859]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T23:10:53.588 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:53 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-22T23:10:53.588 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:53 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-22T23:10:53.588 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:53 smithi069 ceph-mon[21859]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T23:10:53.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:53 smithi149 ceph-mon[23948]: from='client.14516 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:53.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:53 smithi149 ceph-mon[23948]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 2 op/s 2024-04-22T23:10:53.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:53 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-22T23:10:53.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:53 smithi149 ceph-mon[23948]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T23:10:53.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:53 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-22T23:10:53.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:53 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-22T23:10:53.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:53 smithi149 ceph-mon[23948]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T23:10:54.017 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-04-22T23:10:54.503 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: Saving service nfs.foo spec with placement count:2 2024-04-22T23:10:54.503 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:54.503 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:10:54.503 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:54.503 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:10:54.503 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:54.503 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:54.503 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: Creating key for client.nfs.foo.0.0.smithi069.cdmddj 2024-04-22T23:10:54.503 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.cdmddj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T23:10:54.503 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.cdmddj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-22T23:10:54.503 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-22T23:10:54.503 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T23:10:54.504 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T23:10:54.504 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:54.504 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T23:10:54.504 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T23:10:54.504 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.cdmddj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T23:10:54.504 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.cdmddj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T23:10:54.504 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:54.504 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:54 smithi069 ceph-mon[21859]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T23:10:54.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: Saving service nfs.foo spec with placement count:2 2024-04-22T23:10:54.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:54.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:10:54.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:54.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:10:54.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:54.602 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: Creating key for client.nfs.foo.0.0.smithi069.cdmddj 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.cdmddj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.cdmddj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.cdmddj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.cdmddj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:54.603 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:54 smithi149 ceph-mon[23948]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T23:10:55.571 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:55 smithi069 ceph-mon[21859]: Rados config object exists: conf-nfs.foo 2024-04-22T23:10:55.571 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:55 smithi069 ceph-mon[21859]: Creating key for client.nfs.foo.0.0.smithi069.cdmddj-rgw 2024-04-22T23:10:55.571 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:55 smithi069 ceph-mon[21859]: Deploying daemon nfs.foo.0.0.smithi069.cdmddj on smithi069 2024-04-22T23:10:55.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:55 smithi069 ceph-mon[21859]: pgmap v228: 97 pgs: 14 creating+peering, 18 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:55.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:55 smithi069 ceph-mon[21859]: mgrmap e20: smithi069.lyiiqg(active, since 5m), standbys: smithi149.rxqkqk 2024-04-22T23:10:55.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:55 smithi149 ceph-mon[23948]: Rados config object exists: conf-nfs.foo 2024-04-22T23:10:55.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:55 smithi149 ceph-mon[23948]: Creating key for client.nfs.foo.0.0.smithi069.cdmddj-rgw 2024-04-22T23:10:55.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:55 smithi149 ceph-mon[23948]: Deploying daemon nfs.foo.0.0.smithi069.cdmddj on smithi069 2024-04-22T23:10:55.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:55 smithi149 ceph-mon[23948]: pgmap v228: 97 pgs: 14 creating+peering, 18 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:55.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:55 smithi149 ceph-mon[23948]: mgrmap e20: smithi069.lyiiqg(active, since 5m), standbys: smithi149.rxqkqk 2024-04-22T23:10:57.368 INFO:teuthology.orchestra.run.smithi069.stderr:Invalid command: Unexpected argument '--clusterid' 2024-04-22T23:10:57.369 INFO:teuthology.orchestra.run.smithi069.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-04-22T23:10:57.369 INFO:teuthology.orchestra.run.smithi069.stderr:Error EINVAL: invalid command 2024-04-22T23:10:57.526 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:57 smithi069 ceph-mon[21859]: pgmap v230: 97 pgs: 14 creating+peering, 10 unknown, 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:57.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:57 smithi149 ceph-mon[23948]: pgmap v230: 97 pgs: 14 creating+peering, 10 unknown, 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-22T23:10:57.722 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:10:57.722 INFO:teuthology.orchestra.run.smithi069.stdout: "bind": "/fake", 2024-04-22T23:10:57.722 INFO:teuthology.orchestra.run.smithi069.stdout: "fs": "foofs", 2024-04-22T23:10:57.722 INFO:teuthology.orchestra.run.smithi069.stdout: "path": "/", 2024-04-22T23:10:57.723 INFO:teuthology.orchestra.run.smithi069.stdout: "cluster": "foo", 2024-04-22T23:10:57.723 INFO:teuthology.orchestra.run.smithi069.stdout: "mode": "RW" 2024-04-22T23:10:57.723 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:10:58.571 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-04-22T23:10:58.879 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:58 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:58.879 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:58 smithi069 ceph-mon[21859]: Creating key for client.nfs.foo.1.0.smithi149.mxpcoc 2024-04-22T23:10:58.879 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:58 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.mxpcoc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T23:10:58.879 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:58 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.mxpcoc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-22T23:10:58.879 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:58 smithi069 ceph-mon[21859]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-22T23:10:58.879 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:58 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T23:10:58.879 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:58 smithi069 ceph-mon[21859]: from='client.14538 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:58.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:58 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T23:10:58.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:58 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:58.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:58 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-22T23:10:58.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:58 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-22T23:10:58.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:10:58 smithi069 ceph-mon[21859]: pgmap v231: 97 pgs: 9 creating+peering, 88 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-22T23:10:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:58 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:10:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:58 smithi149 ceph-mon[23948]: Creating key for client.nfs.foo.1.0.smithi149.mxpcoc 2024-04-22T23:10:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:58 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.mxpcoc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T23:10:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:58 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.mxpcoc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-22T23:10:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:58 smithi149 ceph-mon[23948]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-22T23:10:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:58 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T23:10:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:58 smithi149 ceph-mon[23948]: from='client.14538 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:10:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:58 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T23:10:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:58 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:10:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:58 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-22T23:10:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:58 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-22T23:10:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:10:58 smithi149 ceph-mon[23948]: pgmap v231: 97 pgs: 9 creating+peering, 88 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-22T23:11:01.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:01 smithi149 ceph-mon[23948]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 130 B/s rd, 261 B/s wr, 0 op/s 2024-04-22T23:11:01.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:01 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T23:11:01.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:01 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T23:11:01.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:01 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.mxpcoc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T23:11:01.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:01 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.mxpcoc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T23:11:01.331 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:01 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:11:01.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:01 smithi069 ceph-mon[21859]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 130 B/s rd, 261 B/s wr, 0 op/s 2024-04-22T23:11:01.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:01 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T23:11:01.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:01 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T23:11:01.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:01 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.mxpcoc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T23:11:01.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:01 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.mxpcoc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T23:11:01.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:01 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:11:02.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:02 smithi149 ceph-mon[23948]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:02.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:02 smithi149 ceph-mon[23948]: Rados config object exists: conf-nfs.foo 2024-04-22T23:11:02.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:02 smithi149 ceph-mon[23948]: Creating key for client.nfs.foo.1.0.smithi149.mxpcoc-rgw 2024-04-22T23:11:02.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:02 smithi149 ceph-mon[23948]: Deploying daemon nfs.foo.1.0.smithi149.mxpcoc on smithi149 2024-04-22T23:11:02.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:02 smithi069 ceph-mon[21859]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:02.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:02 smithi069 ceph-mon[21859]: Rados config object exists: conf-nfs.foo 2024-04-22T23:11:02.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:02 smithi069 ceph-mon[21859]: Creating key for client.nfs.foo.1.0.smithi149.mxpcoc-rgw 2024-04-22T23:11:02.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:02 smithi069 ceph-mon[21859]: Deploying daemon nfs.foo.1.0.smithi149.mxpcoc on smithi149 2024-04-22T23:11:03.369 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:03 smithi069 ceph-mon[21859]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:03.369 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:03 smithi069 ceph-mon[21859]: pgmap v233: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-04-22T23:11:03.369 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:03 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:03.369 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:03 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:11:03.369 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:03 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:11:03.369 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:03 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:11:03.370 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:03 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:03 smithi149 ceph-mon[23948]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:03 smithi149 ceph-mon[23948]: pgmap v233: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-04-22T23:11:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:03 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:03 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:11:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:03 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:11:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:03 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:11:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:03 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:04.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:04 smithi069 ceph-mon[21859]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:04.443 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:04 smithi149 ceph-mon[23948]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:05.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:05 smithi069 ceph-mon[21859]: pgmap v234: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-22T23:11:05.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:05 smithi149 ceph-mon[23948]: pgmap v234: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-22T23:11:06.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:06 smithi069 ceph-mon[21859]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:06.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:06 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:06.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:06 smithi149 ceph-mon[23948]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:06.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:06 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:07.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:07 smithi069 ceph-mon[21859]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:07.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:07 smithi069 ceph-mon[21859]: pgmap v235: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-22T23:11:07.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:07 smithi149 ceph-mon[23948]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:07.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:07 smithi149 ceph-mon[23948]: pgmap v235: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-22T23:11:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]}]: dispatch 2024-04-22T23:11:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:11:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]}]: dispatch 2024-04-22T23:11:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-04-22T23:11:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2024-04-22T23:11:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-04-22T23:11:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:08.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:08.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:11:08.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:11:08.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:11:08.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:08.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:08.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:11:08.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:11:08.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:08.452 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-04-22T23:11:09.133 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-22T23:11:09.142 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T23:11:09.143 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-04-22T23:11:09.169 INFO:teuthology.orchestra.run.smithi069.stderr:+ mkdir /mnt/foo 2024-04-22T23:11:09.186 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-04-22T23:11:09.250 INFO:teuthology.orchestra.run.smithi069.stderr:++ hostname 2024-04-22T23:11:09.250 INFO:teuthology.orchestra.run.smithi069.stderr:+ mount -t nfs smithi069:/fake /mnt/foo -o sync 2024-04-22T23:11:09.381 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:09 smithi069 ceph-mon[21859]: pgmap v236: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-04-22T23:11:09.381 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:09 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]}]': finished 2024-04-22T23:11:09.381 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:09 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-04-22T23:11:09.382 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:09 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-04-22T23:11:09.382 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:09 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-04-22T23:11:09.382 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:09 smithi069 ceph-mon[21859]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T23:11:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:09 smithi149 ceph-mon[23948]: pgmap v236: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-04-22T23:11:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:09 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]}]': finished 2024-04-22T23:11:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:09 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-04-22T23:11:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:09 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-04-22T23:11:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:09 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-04-22T23:11:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:09 smithi149 ceph-mon[23948]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T23:11:09.648 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-04-22T23:11:09.711 INFO:teuthology.orchestra.run.smithi069.stderr:+ echo test 2024-04-22T23:11:09.747 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-04-22T23:11:09.809 INFO:teuthology.orchestra.run.smithi069.stderr:+ sync 2024-04-22T23:11:10.006 INFO:teuthology.run_tasks:Running task parallel... 2024-04-22T23:11:10.016 INFO:teuthology.task.parallel:starting parallel... 2024-04-22T23:11:10.016 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-22T23:11:10.017 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-22T23:11:10.017 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T23:11:10.017 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-22T23:11:10.018 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-22T23:11:10.018 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-04-22T23:11:10.019 INFO:teuthology.task.exec:Executing custom commands... 2024-04-22T23:11:10.019 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T23:11:10.019 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-04-22T23:11:10.131 INFO:teuthology.orchestra.run.smithi069.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-22T23:11:10.131 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:11:10.131 INFO:teuthology.orchestra.run.smithi069.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-04-22T23:11:10.132 INFO:teuthology.orchestra.run.smithi069.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-22T23:11:10.234 INFO:teuthology.orchestra.run.smithi069.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-22T23:11:10.234 INFO:teuthology.orchestra.run.smithi069.stdout:releasing clients 2024-04-22T23:11:10.354 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:10 smithi069 ceph-mon[21859]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:10.354 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:10 smithi069 ceph-mon[21859]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T23:11:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:10 smithi149 ceph-mon[23948]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:10 smithi149 ceph-mon[23948]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T23:11:11.217 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:11 smithi069 ceph-mon[21859]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-04-22T23:11:11.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 154 32.40 MB/sec warmup 1 sec latency 30.442 ms 2024-04-22T23:11:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:11 smithi149 ceph-mon[23948]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-04-22T23:11:12.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 312 28.52 MB/sec warmup 2 sec latency 24.381 ms 2024-04-22T23:11:13.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 491 28.44 MB/sec warmup 3 sec latency 33.183 ms 2024-04-22T23:11:13.296 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:13 smithi069 ceph-mon[21859]: pgmap v240: 97 pgs: 1 peering, 96 active+clean; 15 MiB data, 68 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.9 MiB/s wr, 110 op/s 2024-04-22T23:11:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:13 smithi149 ceph-mon[23948]: pgmap v240: 97 pgs: 1 peering, 96 active+clean; 15 MiB data, 68 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.9 MiB/s wr, 110 op/s 2024-04-22T23:11:13.455 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-22T23:11:14.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 743 26.64 MB/sec warmup 4 sec latency 24.125 ms 2024-04-22T23:11:14.613 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:14 smithi069 ceph-mon[21859]: pgmap v241: 97 pgs: 1 peering, 96 active+clean; 32 MiB data, 73 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.0 MiB/s wr, 151 op/s 2024-04-22T23:11:14.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:14 smithi149 ceph-mon[23948]: pgmap v241: 97 pgs: 1 peering, 96 active+clean; 32 MiB data, 73 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.0 MiB/s wr, 151 op/s 2024-04-22T23:11:15.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 1328 24.27 MB/sec warmup 5 sec latency 24.347 ms 2024-04-22T23:11:16.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 2514 24.17 MB/sec warmup 6 sec latency 16.645 ms 2024-04-22T23:11:17.124 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:17 smithi069 ceph-mon[21859]: pgmap v242: 97 pgs: 97 active+clean; 85 MiB data, 124 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s wr, 286 op/s 2024-04-22T23:11:17.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 3468 26.22 MB/sec warmup 7 sec latency 25.269 ms 2024-04-22T23:11:17.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:17 smithi149 ceph-mon[23948]: pgmap v242: 97 pgs: 97 active+clean; 85 MiB data, 124 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s wr, 286 op/s 2024-04-22T23:11:18.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 4209 25.15 MB/sec warmup 8 sec latency 13.871 ms 2024-04-22T23:11:18.531 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-22T23:11:19.133 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:19 smithi069 ceph-mon[21859]: pgmap v243: 97 pgs: 97 active+clean; 110 MiB data, 164 MiB used, 715 GiB / 715 GiB avail; 206 B/s rd, 12 MiB/s wr, 539 op/s 2024-04-22T23:11:19.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 5235 25.18 MB/sec warmup 9 sec latency 12.028 ms 2024-04-22T23:11:19.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:19 smithi149 ceph-mon[23948]: pgmap v243: 97 pgs: 97 active+clean; 110 MiB data, 164 MiB used, 715 GiB / 715 GiB avail; 206 B/s rd, 12 MiB/s wr, 539 op/s 2024-04-22T23:11:20.233 INFO:teuthology.orchestra.run.smithi069.stdout: 5 6080 24.03 MB/sec warmup 10 sec latency 39.682 ms 2024-04-22T23:11:21.190 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:21 smithi069 ceph-mon[21859]: pgmap v244: 97 pgs: 97 active+clean; 129 MiB data, 233 MiB used, 715 GiB / 715 GiB avail; 187 B/s rd, 14 MiB/s wr, 786 op/s 2024-04-22T23:11:21.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 6700 24.43 MB/sec warmup 11 sec latency 156.973 ms 2024-04-22T23:11:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:21 smithi149 ceph-mon[23948]: pgmap v244: 97 pgs: 97 active+clean; 129 MiB data, 233 MiB used, 715 GiB / 715 GiB avail; 187 B/s rd, 14 MiB/s wr, 786 op/s 2024-04-22T23:11:22.233 INFO:teuthology.orchestra.run.smithi069.stdout: 5 7386 24.56 MB/sec warmup 12 sec latency 38.967 ms 2024-04-22T23:11:23.212 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:23 smithi069 ceph-mon[21859]: pgmap v245: 97 pgs: 97 active+clean; 137 MiB data, 279 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 15 MiB/s wr, 953 op/s 2024-04-22T23:11:23.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 7886 23.00 MB/sec warmup 13 sec latency 30.658 ms 2024-04-22T23:11:23.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:23 smithi149 ceph-mon[23948]: pgmap v245: 97 pgs: 97 active+clean; 137 MiB data, 279 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 15 MiB/s wr, 953 op/s 2024-04-22T23:11:24.233 INFO:teuthology.orchestra.run.smithi069.stdout: 5 8475 22.33 MB/sec warmup 14 sec latency 32.294 ms 2024-04-22T23:11:24.627 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph mgr module enable nfs --force' 2024-04-22T23:11:24.732 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:24 smithi069 ceph-mon[21859]: pgmap v246: 97 pgs: 97 active+clean; 139 MiB data, 290 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 14 MiB/s wr, 926 op/s 2024-04-22T23:11:24.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:24 smithi149 ceph-mon[23948]: pgmap v246: 97 pgs: 97 active+clean; 139 MiB data, 290 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 14 MiB/s wr, 926 op/s 2024-04-22T23:11:25.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 9320 21.84 MB/sec warmup 15 sec latency 23.185 ms 2024-04-22T23:11:26.232 INFO:teuthology.orchestra.run.smithi069.stdout: 5 9883 21.60 MB/sec warmup 16 sec latency 31.882 ms 2024-04-22T23:11:27.237 INFO:teuthology.orchestra.run.smithi069.stdout: 5 10533 22.02 MB/sec warmup 17 sec latency 23.205 ms 2024-04-22T23:11:27.287 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:27 smithi069 ceph-mon[21859]: pgmap v247: 97 pgs: 97 active+clean; 146 MiB data, 357 MiB used, 715 GiB / 715 GiB avail; 9.8 KiB/s rd, 15 MiB/s wr, 1.10k op/s 2024-04-22T23:11:27.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:27 smithi149 ceph-mon[23948]: pgmap v247: 97 pgs: 97 active+clean; 146 MiB data, 357 MiB used, 715 GiB / 715 GiB avail; 9.8 KiB/s rd, 15 MiB/s wr, 1.10k op/s 2024-04-22T23:11:28.237 INFO:teuthology.orchestra.run.smithi069.stdout: 5 11080 21.73 MB/sec warmup 18 sec latency 21.904 ms 2024-04-22T23:11:29.123 INFO:teuthology.orchestra.run.smithi069.stderr:module 'nfs' is already enabled 2024-04-22T23:11:29.237 INFO:teuthology.orchestra.run.smithi069.stdout: 5 11676 21.05 MB/sec warmup 19 sec latency 18.268 ms 2024-04-22T23:11:29.295 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:29 smithi069 ceph-mon[21859]: pgmap v248: 97 pgs: 97 active+clean; 147 MiB data, 393 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 12 MiB/s wr, 1.05k op/s 2024-04-22T23:11:29.295 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:29 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3491146665' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-22T23:11:29.296 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:29 smithi069 ceph-mon[21859]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-22T23:11:29.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:29 smithi149 ceph-mon[23948]: pgmap v248: 97 pgs: 97 active+clean; 147 MiB data, 393 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 12 MiB/s wr, 1.05k op/s 2024-04-22T23:11:29.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:29 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/3491146665' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-22T23:11:29.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:29 smithi149 ceph-mon[23948]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-22T23:11:30.237 INFO:teuthology.orchestra.run.smithi069.stdout: 5 12380 20.93 MB/sec warmup 20 sec latency 29.219 ms 2024-04-22T23:11:30.299 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:30 smithi069 ceph-mon[21859]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-22T23:11:30.299 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:30 smithi069 ceph-mon[21859]: mgrmap e21: smithi069.lyiiqg(active, since 6m), standbys: smithi149.rxqkqk 2024-04-22T23:11:30.305 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-22T23:11:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:30 smithi149 ceph-mon[23948]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-22T23:11:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:30 smithi149 ceph-mon[23948]: mgrmap e21: smithi069.lyiiqg(active, since 6m), standbys: smithi149.rxqkqk 2024-04-22T23:11:31.237 INFO:teuthology.orchestra.run.smithi069.stdout: 5 13134 20.54 MB/sec warmup 21 sec latency 32.157 ms 2024-04-22T23:11:31.306 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:31 smithi069 ceph-mon[21859]: pgmap v249: 97 pgs: 97 active+clean; 156 MiB data, 433 MiB used, 715 GiB / 715 GiB avail; 9.8 KiB/s rd, 12 MiB/s wr, 1.07k op/s 2024-04-22T23:11:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:31 smithi149 ceph-mon[23948]: pgmap v249: 97 pgs: 97 active+clean; 156 MiB data, 433 MiB used, 715 GiB / 715 GiB avail; 9.8 KiB/s rd, 12 MiB/s wr, 1.07k op/s 2024-04-22T23:11:32.237 INFO:teuthology.orchestra.run.smithi069.stdout: 5 13685 20.76 MB/sec warmup 22 sec latency 25.231 ms 2024-04-22T23:11:33.238 INFO:teuthology.orchestra.run.smithi069.stdout: 5 14250 20.77 MB/sec warmup 23 sec latency 39.237 ms 2024-04-22T23:11:33.328 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:33 smithi069 ceph-mon[21859]: pgmap v250: 97 pgs: 97 active+clean; 156 MiB data, 458 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.5 MiB/s wr, 914 op/s 2024-04-22T23:11:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:33 smithi149 ceph-mon[23948]: pgmap v250: 97 pgs: 97 active+clean; 156 MiB data, 458 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.5 MiB/s wr, 914 op/s 2024-04-22T23:11:33.686 INFO:teuthology.orchestra.run.smithi069.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:43be020184947e53516056c9931e1ac5bdbbb1a5 2024-04-22T23:11:34.238 INFO:teuthology.orchestra.run.smithi069.stdout: 5 14723 20.32 MB/sec warmup 24 sec latency 24.644 ms 2024-04-22T23:11:34.835 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:34 smithi069 ceph-mon[21859]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:43be020184947e53516056c9931e1ac5bdbbb1a5", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:34.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:34 smithi069 ceph-mon[21859]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:43be020184947e53516056c9931e1ac5bdbbb1a5 2024-04-22T23:11:34.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:34 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:34.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:34 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:11:34.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:34 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:11:34.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:34 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:11:34.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:34 smithi069 ceph-mon[21859]: pgmap v251: 97 pgs: 97 active+clean; 155 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 704 op/s 2024-04-22T23:11:34.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:34 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:34.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:34 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:34.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:34 smithi069 ceph-mon[21859]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:43be020184947e53516056c9931e1ac5bdbbb1a5 2024-04-22T23:11:34.839 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-22T23:11:34.839 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T23:11:34.840 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-22T23:11:34.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:34 smithi149 ceph-mon[23948]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:43be020184947e53516056c9931e1ac5bdbbb1a5", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:34.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:34 smithi149 ceph-mon[23948]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:43be020184947e53516056c9931e1ac5bdbbb1a5 2024-04-22T23:11:34.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:34 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:34.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:34 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:11:34.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:34 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:11:34.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:34 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:11:34.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:34 smithi149 ceph-mon[23948]: pgmap v251: 97 pgs: 97 active+clean; 155 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 704 op/s 2024-04-22T23:11:34.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:34 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:34.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:34 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:11:34.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:34 smithi149 ceph-mon[23948]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:43be020184947e53516056c9931e1ac5bdbbb1a5 2024-04-22T23:11:35.238 INFO:teuthology.orchestra.run.smithi069.stdout: 5 15421 20.00 MB/sec warmup 25 sec latency 18.330 ms 2024-04-22T23:11:36.238 INFO:teuthology.orchestra.run.smithi069.stdout: 5 16534 20.10 MB/sec warmup 26 sec latency 12.990 ms 2024-04-22T23:11:37.238 INFO:teuthology.orchestra.run.smithi069.stdout: 5 17476 20.68 MB/sec warmup 27 sec latency 22.748 ms 2024-04-22T23:11:37.348 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:37 smithi069 ceph-mon[21859]: pgmap v252: 97 pgs: 97 active+clean; 163 MiB data, 494 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 9.8 MiB/s wr, 911 op/s 2024-04-22T23:11:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:37 smithi149 ceph-mon[23948]: pgmap v252: 97 pgs: 97 active+clean; 163 MiB data, 494 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 9.8 MiB/s wr, 911 op/s 2024-04-22T23:11:38.238 INFO:teuthology.orchestra.run.smithi069.stdout: 5 18145 20.78 MB/sec warmup 28 sec latency 21.002 ms 2024-04-22T23:11:39.047 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:11:39.238 INFO:teuthology.orchestra.run.smithi069.stdout: 5 18703 20.35 MB/sec warmup 29 sec latency 22.141 ms 2024-04-22T23:11:39.358 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:39 smithi069 ceph-mon[21859]: pgmap v253: 97 pgs: 97 active+clean; 165 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.7 MiB/s wr, 764 op/s 2024-04-22T23:11:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:39 smithi149 ceph-mon[23948]: pgmap v253: 97 pgs: 97 active+clean; 165 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.7 MiB/s wr, 764 op/s 2024-04-22T23:11:40.101 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:11:40.101 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (5m) 32s ago 6m 24.4M - ba2b418f427c e0aa152f63bd 2024-04-22T23:11:40.101 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (6m) 32s ago 6m 6400k - 17.2.0 e1d6a67b021e f29a69890fd2 2024-04-22T23:11:40.101 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (6m) 34s ago 6m 6400k - 17.2.0 e1d6a67b021e d3576856bdfd 2024-04-22T23:11:40.101 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (5m) 32s ago 6m 52.4M - 8.3.5 dad864ee21e9 c5880e24c661 2024-04-22T23:11:40.101 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (64s) 32s ago 63s 13.4M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:11:40.101 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (61s) 34s ago 61s 12.1M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:11:40.101 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:9283 running (8m) 32s ago 8m 460M - 17.2.0 e1d6a67b021e 98c4390893c8 2024-04-22T23:11:40.101 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283 running (6m) 34s ago 6m 413M - 17.2.0 e1d6a67b021e fa2d6964da01 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (8m) 32s ago 8m 48.3M 2048M 17.2.0 e1d6a67b021e 008ad8fe38c6 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (6m) 34s ago 6m 37.7M 2048M 17.2.0 e1d6a67b021e 1a87df28e650 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (42s) 32s ago 42s 49.9M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (37s) 34s ago 37s 50.7M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (6m) 32s ago 6m 18.9M - 1dbe0e931976 1175a40c25d7 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (6m) 34s ago 6m 17.1M - 1dbe0e931976 fd4ddbc4e0f2 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (5m) 32s ago 5m 45.4M 3216M 17.2.0 e1d6a67b021e 777db7a3d809 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (4m) 32s ago 4m 42.5M 3216M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (4m) 32s ago 4m 43.0M 3216M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (4m) 32s ago 4m 42.5M 3216M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (3m) 34s ago 3m 39.8M 1166M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (3m) 34s ago 3m 42.0M 1166M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:11:40.102 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (2m) 34s ago 2m 39.8M 1166M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:11:40.103 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (2m) 34s ago 2m 41.1M 1166M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:11:40.103 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (5m) 32s ago 5m 62.4M - 514e6a882f6e b8ea50a9c418 2024-04-22T23:11:40.238 INFO:teuthology.orchestra.run.smithi069.stdout: 5 19464 20.31 MB/sec warmup 30 sec latency 20.568 ms 2024-04-22T23:11:40.276 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:40 smithi069 ceph-mon[21859]: from='client.24339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:40.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:40 smithi149 ceph-mon[23948]: from='client.24339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:40.747 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:11:40.747 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:11:40.747 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:11:40.747 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:11:40.747 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:11:40.747 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:11:40.747 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:11:40.747 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:11:40.747 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T23:11:40.748 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:11:40.748 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:11:40.748 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:11:40.748 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:11:40.748 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:11:40.748 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-22T23:11:40.748 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:11:40.748 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:11:41.238 INFO:teuthology.orchestra.run.smithi069.stdout: 5 20283 20.12 MB/sec warmup 31 sec latency 23.910 ms 2024-04-22T23:11:41.257 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:11:41.257 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:43be020184947e53516056c9931e1ac5bdbbb1a5", 2024-04-22T23:11:41.257 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:11:41.258 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [], 2024-04-22T23:11:41.258 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "", 2024-04-22T23:11:41.258 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:43be020184947e53516056c9931e1ac5bdbbb1a5 image" 2024-04-22T23:11:41.258 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:11:41.368 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:41 smithi069 ceph-mon[21859]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:41.368 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:41 smithi069 ceph-mon[21859]: pgmap v254: 97 pgs: 97 active+clean; 171 MiB data, 578 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 925 op/s 2024-04-22T23:11:41.368 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:41 smithi069 ceph-mon[21859]: from='client.24345 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:41.369 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:41 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3704945489' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:11:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:41 smithi149 ceph-mon[23948]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:41 smithi149 ceph-mon[23948]: pgmap v254: 97 pgs: 97 active+clean; 171 MiB data, 578 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 925 op/s 2024-04-22T23:11:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:41 smithi149 ceph-mon[23948]: from='client.24345 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:41 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/3704945489' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:11:41.890 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:11:42.138 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:42 smithi069 ceph-mon[21859]: from='client.24353 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:42.138 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:42 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3439466539' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:11:42.238 INFO:teuthology.orchestra.run.smithi069.stdout: 5 21240 20.67 MB/sec warmup 32 sec latency 25.170 ms 2024-04-22T23:11:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:42 smithi149 ceph-mon[23948]: from='client.24353 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:11:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:42 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/3439466539' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:11:43.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:43 smithi069 ceph-mon[21859]: pgmap v255: 97 pgs: 97 active+clean; 173 MiB data, 607 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 9.1 MiB/s wr, 815 op/s 2024-04-22T23:11:43.238 INFO:teuthology.orchestra.run.smithi069.stdout: 5 21935 20.63 MB/sec warmup 33 sec latency 21.540 ms 2024-04-22T23:11:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:43 smithi149 ceph-mon[23948]: pgmap v255: 97 pgs: 97 active+clean; 173 MiB data, 607 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 9.1 MiB/s wr, 815 op/s 2024-04-22T23:11:44.239 INFO:teuthology.orchestra.run.smithi069.stdout: 5 22660 20.44 MB/sec warmup 34 sec latency 15.780 ms 2024-04-22T23:11:44.655 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:44 smithi069 ceph-mon[21859]: pgmap v256: 97 pgs: 97 active+clean; 178 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 810 op/s 2024-04-22T23:11:44.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:44 smithi149 ceph-mon[23948]: pgmap v256: 97 pgs: 97 active+clean; 178 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 810 op/s 2024-04-22T23:11:45.239 INFO:teuthology.orchestra.run.smithi069.stdout: 5 23626 20.41 MB/sec warmup 35 sec latency 29.752 ms 2024-04-22T23:11:46.239 INFO:teuthology.orchestra.run.smithi069.stdout: 5 24547 20.80 MB/sec warmup 36 sec latency 34.931 ms 2024-04-22T23:11:47.164 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:47 smithi069 ceph-mon[21859]: pgmap v257: 97 pgs: 97 active+clean; 191 MiB data, 692 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.11k op/s 2024-04-22T23:11:47.239 INFO:teuthology.orchestra.run.smithi069.stdout: 5 25230 20.86 MB/sec warmup 37 sec latency 42.311 ms 2024-04-22T23:11:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:47 smithi149 ceph-mon[23948]: pgmap v257: 97 pgs: 97 active+clean; 191 MiB data, 692 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.11k op/s 2024-04-22T23:11:48.240 INFO:teuthology.orchestra.run.smithi069.stdout: 5 25715 20.54 MB/sec warmup 38 sec latency 23.031 ms 2024-04-22T23:11:49.172 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:49 smithi069 ceph-mon[21859]: pgmap v258: 97 pgs: 97 active+clean; 195 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 922 op/s 2024-04-22T23:11:49.240 INFO:teuthology.orchestra.run.smithi069.stdout: 5 26411 20.37 MB/sec warmup 39 sec latency 19.190 ms 2024-04-22T23:11:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:49 smithi149 ceph-mon[23948]: pgmap v258: 97 pgs: 97 active+clean; 195 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 922 op/s 2024-04-22T23:11:50.240 INFO:teuthology.orchestra.run.smithi069.stdout: 5 27364 20.35 MB/sec warmup 40 sec latency 17.832 ms 2024-04-22T23:11:51.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:51 smithi069 ceph-mon[21859]: pgmap v259: 97 pgs: 97 active+clean; 202 MiB data, 775 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.13k op/s 2024-04-22T23:11:51.240 INFO:teuthology.orchestra.run.smithi069.stdout: 5 28234 20.74 MB/sec warmup 41 sec latency 20.090 ms 2024-04-22T23:11:51.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:51 smithi149 ceph-mon[23948]: pgmap v259: 97 pgs: 97 active+clean; 202 MiB data, 775 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.13k op/s 2024-04-22T23:11:52.240 INFO:teuthology.orchestra.run.smithi069.stdout: 5 29058 20.77 MB/sec warmup 42 sec latency 25.421 ms 2024-04-22T23:11:53.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:53 smithi069 ceph-mon[21859]: pgmap v260: 97 pgs: 97 active+clean; 200 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 11 MiB/s wr, 981 op/s 2024-04-22T23:11:53.240 INFO:teuthology.orchestra.run.smithi069.stdout: 5 29953 20.72 MB/sec warmup 43 sec latency 18.527 ms 2024-04-22T23:11:53.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:53 smithi149 ceph-mon[23948]: pgmap v260: 97 pgs: 97 active+clean; 200 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 11 MiB/s wr, 981 op/s 2024-04-22T23:11:54.240 INFO:teuthology.orchestra.run.smithi069.stdout: 5 30931 20.69 MB/sec warmup 44 sec latency 26.746 ms 2024-04-22T23:11:54.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:54 smithi069 ceph-mon[21859]: pgmap v261: 97 pgs: 97 active+clean; 209 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-22T23:11:54.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:54 smithi149 ceph-mon[23948]: pgmap v261: 97 pgs: 97 active+clean; 209 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-22T23:11:55.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 31778 21.02 MB/sec warmup 45 sec latency 20.045 ms 2024-04-22T23:11:56.240 INFO:teuthology.orchestra.run.smithi069.stdout: 5 32468 21.00 MB/sec warmup 46 sec latency 19.872 ms 2024-04-22T23:11:57.152 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:57 smithi069 ceph-mon[21859]: pgmap v262: 97 pgs: 97 active+clean; 212 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-04-22T23:11:57.240 INFO:teuthology.orchestra.run.smithi069.stdout: 5 33150 20.77 MB/sec warmup 47 sec latency 16.648 ms 2024-04-22T23:11:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:57 smithi149 ceph-mon[23948]: pgmap v262: 97 pgs: 97 active+clean; 212 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-04-22T23:11:58.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 34255 20.87 MB/sec warmup 48 sec latency 14.838 ms 2024-04-22T23:11:59.160 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:11:59 smithi069 ceph-mon[21859]: pgmap v263: 97 pgs: 97 active+clean; 225 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-04-22T23:11:59.240 INFO:teuthology.orchestra.run.smithi069.stdout: 5 35207 21.13 MB/sec warmup 49 sec latency 20.593 ms 2024-04-22T23:11:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:11:59 smithi149 ceph-mon[23948]: pgmap v263: 97 pgs: 97 active+clean; 225 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-04-22T23:12:00.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 35883 21.19 MB/sec warmup 50 sec latency 25.460 ms 2024-04-22T23:12:01.170 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:01 smithi069 ceph-mon[21859]: pgmap v264: 97 pgs: 97 active+clean; 223 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.10k op/s 2024-04-22T23:12:01.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 36392 20.93 MB/sec warmup 51 sec latency 41.808 ms 2024-04-22T23:12:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:01 smithi149 ceph-mon[23948]: pgmap v264: 97 pgs: 97 active+clean; 223 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.10k op/s 2024-04-22T23:12:02.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 37049 20.81 MB/sec warmup 52 sec latency 24.731 ms 2024-04-22T23:12:03.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:03 smithi069 ceph-mon[21859]: pgmap v265: 97 pgs: 97 active+clean; 231 MiB data, 1018 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 12 MiB/s wr, 1.07k op/s 2024-04-22T23:12:03.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 37672 20.67 MB/sec warmup 53 sec latency 28.492 ms 2024-04-22T23:12:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:03 smithi149 ceph-mon[23948]: pgmap v265: 97 pgs: 97 active+clean; 231 MiB data, 1018 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 12 MiB/s wr, 1.07k op/s 2024-04-22T23:12:04.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 38175 20.52 MB/sec warmup 54 sec latency 47.515 ms 2024-04-22T23:12:04.727 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:04 smithi069 ceph-mon[21859]: pgmap v266: 97 pgs: 97 active+clean; 231 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-04-22T23:12:04.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:04 smithi149 ceph-mon[23948]: pgmap v266: 97 pgs: 97 active+clean; 231 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-04-22T23:12:05.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 38687 20.55 MB/sec warmup 55 sec latency 30.613 ms 2024-04-22T23:12:06.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 39357 20.60 MB/sec warmup 56 sec latency 21.596 ms 2024-04-22T23:12:07.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 39779 20.34 MB/sec warmup 57 sec latency 29.992 ms 2024-04-22T23:12:07.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:07 smithi069 ceph-mon[21859]: pgmap v267: 97 pgs: 97 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 11 MiB/s wr, 972 op/s 2024-04-22T23:12:07.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:07 smithi149 ceph-mon[23948]: pgmap v267: 97 pgs: 97 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 11 MiB/s wr, 972 op/s 2024-04-22T23:12:08.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 40371 20.19 MB/sec warmup 58 sec latency 33.137 ms 2024-04-22T23:12:08.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:12:08.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:12:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:12:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:12:09.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 41081 20.16 MB/sec warmup 59 sec latency 61.596 ms 2024-04-22T23:12:09.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:09 smithi069 ceph-mon[21859]: pgmap v268: 97 pgs: 97 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-22T23:12:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:09 smithi149 ceph-mon[23948]: pgmap v268: 97 pgs: 97 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-22T23:12:10.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 41754 20.10 MB/sec warmup 60 sec latency 43.788 ms 2024-04-22T23:12:11.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 42263 20.07 MB/sec warmup 61 sec latency 25.630 ms 2024-04-22T23:12:11.288 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:11 smithi069 ceph-mon[21859]: pgmap v269: 97 pgs: 97 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 825 op/s 2024-04-22T23:12:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:11 smithi149 ceph-mon[23948]: pgmap v269: 97 pgs: 97 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 825 op/s 2024-04-22T23:12:12.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 42944 20.15 MB/sec warmup 62 sec latency 19.287 ms 2024-04-22T23:12:12.451 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:12:13.241 INFO:teuthology.orchestra.run.smithi069.stdout: 5 43381 19.97 MB/sec warmup 63 sec latency 32.320 ms 2024-04-22T23:12:13.297 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:13 smithi069 ceph-mon[21859]: pgmap v270: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.8 MiB/s wr, 893 op/s 2024-04-22T23:12:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:13 smithi149 ceph-mon[23948]: pgmap v270: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.8 MiB/s wr, 893 op/s 2024-04-22T23:12:13.533 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:12:13.533 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (6m) 65s ago 7m 24.4M - ba2b418f427c e0aa152f63bd 2024-04-22T23:12:13.533 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (7m) 65s ago 7m 6400k - 17.2.0 e1d6a67b021e f29a69890fd2 2024-04-22T23:12:13.533 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (6m) 67s ago 6m 6400k - 17.2.0 e1d6a67b021e d3576856bdfd 2024-04-22T23:12:13.533 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (5m) 65s ago 7m 52.4M - 8.3.5 dad864ee21e9 c5880e24c661 2024-04-22T23:12:13.533 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (98s) 65s ago 97s 13.4M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:12:13.533 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (95s) 67s ago 94s 12.1M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:12:13.533 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:9283 running (8m) 65s ago 8m 460M - 17.2.0 e1d6a67b021e 98c4390893c8 2024-04-22T23:12:13.533 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283 running (6m) 67s ago 6m 413M - 17.2.0 e1d6a67b021e fa2d6964da01 2024-04-22T23:12:13.533 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (8m) 65s ago 8m 48.3M 2048M 17.2.0 e1d6a67b021e 008ad8fe38c6 2024-04-22T23:12:13.534 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (6m) 67s ago 6m 37.7M 2048M 17.2.0 e1d6a67b021e 1a87df28e650 2024-04-22T23:12:13.534 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (76s) 65s ago 75s 49.9M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:12:13.534 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (71s) 67s ago 70s 50.7M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:12:13.534 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (7m) 65s ago 7m 18.9M - 1dbe0e931976 1175a40c25d7 2024-04-22T23:12:13.534 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (6m) 67s ago 6m 17.1M - 1dbe0e931976 fd4ddbc4e0f2 2024-04-22T23:12:13.534 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (6m) 65s ago 6m 45.4M 3216M 17.2.0 e1d6a67b021e 777db7a3d809 2024-04-22T23:12:13.534 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (5m) 65s ago 5m 42.5M 3216M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:12:13.534 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (5m) 65s ago 5m 43.0M 3216M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:12:13.534 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (4m) 65s ago 4m 42.5M 3216M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:12:13.534 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (4m) 67s ago 4m 39.8M 1166M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:12:13.534 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (3m) 67s ago 3m 42.0M 1166M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:12:13.534 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (3m) 67s ago 3m 39.8M 1166M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:12:13.535 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (3m) 67s ago 3m 41.1M 1166M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:12:13.535 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (6m) 65s ago 6m 62.4M - 514e6a882f6e b8ea50a9c418 2024-04-22T23:12:14.180 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:12:14.181 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:12:14.181 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:12:14.181 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:12:14.181 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:12:14.181 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:12:14.181 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:12:14.181 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:12:14.181 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T23:12:14.181 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:12:14.181 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:12:14.181 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:12:14.181 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:12:14.182 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:12:14.182 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-22T23:12:14.182 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:12:14.182 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:12:14.242 INFO:teuthology.orchestra.run.smithi069.stdout: 5 43971 19.80 MB/sec warmup 64 sec latency 46.485 ms 2024-04-22T23:12:14.288 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:14 smithi069 ceph-mon[21859]: from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:14.288 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:14 smithi069 ceph-mon[21859]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:14 smithi149 ceph-mon[23948]: from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:14 smithi149 ceph-mon[23948]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:14.733 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:12:14.733 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:43be020184947e53516056c9931e1ac5bdbbb1a5", 2024-04-22T23:12:14.734 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:12:14.734 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [], 2024-04-22T23:12:14.734 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "", 2024-04-22T23:12:14.734 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:43be020184947e53516056c9931e1ac5bdbbb1a5 image" 2024-04-22T23:12:14.734 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:12:15.242 INFO:teuthology.orchestra.run.smithi069.stdout: 5 44611 19.77 MB/sec warmup 65 sec latency 23.151 ms 2024-04-22T23:12:15.299 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:15 smithi069 ceph-mon[21859]: from='client.24369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:15.299 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:15 smithi069 ceph-mon[21859]: pgmap v271: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 791 op/s 2024-04-22T23:12:15.300 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:15 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2983998166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:12:15.350 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:12:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:15 smithi149 ceph-mon[23948]: from='client.24369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:15 smithi149 ceph-mon[23948]: pgmap v271: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 791 op/s 2024-04-22T23:12:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:15 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2983998166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:12:16.243 INFO:teuthology.orchestra.run.smithi069.stdout: 5 45225 19.67 MB/sec warmup 66 sec latency 44.156 ms 2024-04-22T23:12:16.304 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:16 smithi069 ceph-mon[21859]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:16.304 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:16 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2994400412' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:12:16.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:16 smithi149 ceph-mon[23948]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:16.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:16 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2994400412' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:12:17.243 INFO:teuthology.orchestra.run.smithi069.stdout: 5 45770 19.68 MB/sec warmup 67 sec latency 27.333 ms 2024-04-22T23:12:17.314 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:17 smithi069 ceph-mon[21859]: pgmap v272: 97 pgs: 97 active+clean; 257 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.1 MiB/s wr, 863 op/s 2024-04-22T23:12:17.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:17 smithi149 ceph-mon[23948]: pgmap v272: 97 pgs: 97 active+clean; 257 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.1 MiB/s wr, 863 op/s 2024-04-22T23:12:18.243 INFO:teuthology.orchestra.run.smithi069.stdout: 5 46486 19.78 MB/sec warmup 68 sec latency 18.485 ms 2024-04-22T23:12:19.243 INFO:teuthology.orchestra.run.smithi069.stdout: 5 47084 19.65 MB/sec warmup 69 sec latency 20.765 ms 2024-04-22T23:12:19.326 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:19 smithi069 ceph-mon[21859]: pgmap v273: 97 pgs: 97 active+clean; 257 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 8.6 MiB/s wr, 856 op/s 2024-04-22T23:12:19.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:19 smithi149 ceph-mon[23948]: pgmap v273: 97 pgs: 97 active+clean; 257 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 8.6 MiB/s wr, 856 op/s 2024-04-22T23:12:20.245 INFO:teuthology.orchestra.run.smithi069.stdout: 5 47931 19.61 MB/sec warmup 70 sec latency 18.344 ms 2024-04-22T23:12:21.245 INFO:teuthology.orchestra.run.smithi069.stdout: 5 48904 19.70 MB/sec warmup 71 sec latency 23.185 ms 2024-04-22T23:12:21.335 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:21 smithi069 ceph-mon[21859]: pgmap v274: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.1 MiB/s wr, 841 op/s 2024-04-22T23:12:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:21 smithi149 ceph-mon[23948]: pgmap v274: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.1 MiB/s wr, 841 op/s 2024-04-22T23:12:22.247 INFO:teuthology.orchestra.run.smithi069.stdout: 5 49773 19.88 MB/sec warmup 72 sec latency 24.112 ms 2024-04-22T23:12:23.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 50503 19.86 MB/sec warmup 73 sec latency 19.603 ms 2024-04-22T23:12:23.343 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:23 smithi069 ceph-mon[21859]: pgmap v275: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.2 MiB/s wr, 931 op/s 2024-04-22T23:12:23.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:23 smithi149 ceph-mon[23948]: pgmap v275: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.2 MiB/s wr, 931 op/s 2024-04-22T23:12:24.245 INFO:teuthology.orchestra.run.smithi069.stdout: 5 51371 19.80 MB/sec warmup 74 sec latency 17.522 ms 2024-04-22T23:12:25.245 INFO:teuthology.orchestra.run.smithi069.stdout: 5 52027 19.75 MB/sec warmup 75 sec latency 22.751 ms 2024-04-22T23:12:25.353 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:25 smithi069 ceph-mon[21859]: pgmap v276: 97 pgs: 97 active+clean; 279 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 9.1 MiB/s wr, 870 op/s 2024-04-22T23:12:25.353 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:25 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:12:25.353 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:25 smithi069 ceph-mon[21859]: Upgrade: Target is version 19.0.0-3252-g43be0201 (unknown) 2024-04-22T23:12:25.354 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:25 smithi069 ceph-mon[21859]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c'] 2024-04-22T23:12:25.354 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:25 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:12:25.354 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:25 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:12:25.354 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:25 smithi069 ceph-mon[21859]: Upgrade: Need to upgrade myself (mgr.smithi069.lyiiqg) 2024-04-22T23:12:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:25 smithi149 ceph-mon[23948]: pgmap v276: 97 pgs: 97 active+clean; 279 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 9.1 MiB/s wr, 870 op/s 2024-04-22T23:12:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:25 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:12:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:25 smithi149 ceph-mon[23948]: Upgrade: Target is version 19.0.0-3252-g43be0201 (unknown) 2024-04-22T23:12:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:25 smithi149 ceph-mon[23948]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c'] 2024-04-22T23:12:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:25 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:12:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:25 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:12:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:25 smithi149 ceph-mon[23948]: Upgrade: Need to upgrade myself (mgr.smithi069.lyiiqg) 2024-04-22T23:12:26.245 INFO:teuthology.orchestra.run.smithi069.stdout: 5 52643 19.76 MB/sec warmup 76 sec latency 36.745 ms 2024-04-22T23:12:26.365 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:26 smithi069 ceph-mon[21859]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c on smithi149 2024-04-22T23:12:26.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:26 smithi149 ceph-mon[23948]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c on smithi149 2024-04-22T23:12:27.245 INFO:teuthology.orchestra.run.smithi069.stdout: 5 53303 19.83 MB/sec warmup 77 sec latency 32.493 ms 2024-04-22T23:12:27.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:27 smithi069 ceph-mon[21859]: pgmap v277: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-04-22T23:12:27.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:27 smithi149 ceph-mon[23948]: pgmap v277: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-04-22T23:12:28.245 INFO:teuthology.orchestra.run.smithi069.stdout: 5 53920 19.77 MB/sec warmup 78 sec latency 23.107 ms 2024-04-22T23:12:29.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 54626 19.68 MB/sec warmup 79 sec latency 51.097 ms 2024-04-22T23:12:29.427 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:29 smithi069 ceph-mon[21859]: pgmap v278: 97 pgs: 97 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 970 op/s 2024-04-22T23:12:29.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:29 smithi149 ceph-mon[23948]: pgmap v278: 97 pgs: 97 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 970 op/s 2024-04-22T23:12:30.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 55426 19.68 MB/sec warmup 80 sec latency 31.739 ms 2024-04-22T23:12:31.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 56170 19.72 MB/sec warmup 81 sec latency 18.778 ms 2024-04-22T23:12:31.411 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:31 smithi069 ceph-mon[21859]: pgmap v279: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 992 op/s 2024-04-22T23:12:31.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:31 smithi149 ceph-mon[23948]: pgmap v279: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 992 op/s 2024-04-22T23:12:32.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 56839 19.78 MB/sec warmup 82 sec latency 62.107 ms 2024-04-22T23:12:33.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 57424 19.72 MB/sec warmup 83 sec latency 17.994 ms 2024-04-22T23:12:33.420 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:33 smithi069 ceph-mon[21859]: pgmap v280: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 10 MiB/s wr, 934 op/s 2024-04-22T23:12:33.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:33 smithi149 ceph-mon[23948]: pgmap v280: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 10 MiB/s wr, 934 op/s 2024-04-22T23:12:34.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 57927 19.57 MB/sec warmup 84 sec latency 42.588 ms 2024-04-22T23:12:34.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:34 smithi069 ceph-mon[21859]: pgmap v281: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.8 MiB/s wr, 918 op/s 2024-04-22T23:12:34.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:34 smithi149 ceph-mon[23948]: pgmap v281: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.8 MiB/s wr, 918 op/s 2024-04-22T23:12:35.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 58594 19.53 MB/sec warmup 85 sec latency 40.474 ms 2024-04-22T23:12:36.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 59244 19.46 MB/sec warmup 86 sec latency 52.509 ms 2024-04-22T23:12:37.239 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:37 smithi069 ceph-mon[21859]: pgmap v282: 97 pgs: 97 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 907 op/s 2024-04-22T23:12:37.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 59976 19.58 MB/sec warmup 87 sec latency 18.996 ms 2024-04-22T23:12:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:37 smithi149 ceph-mon[23948]: pgmap v282: 97 pgs: 97 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 907 op/s 2024-04-22T23:12:38.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 60673 19.63 MB/sec warmup 88 sec latency 21.462 ms 2024-04-22T23:12:39.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 61251 19.52 MB/sec warmup 89 sec latency 21.756 ms 2024-04-22T23:12:39.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:39 smithi069 ceph-mon[21859]: pgmap v283: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.0 MiB/s wr, 792 op/s 2024-04-22T23:12:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:39 smithi149 ceph-mon[23948]: pgmap v283: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.0 MiB/s wr, 792 op/s 2024-04-22T23:12:40.246 INFO:teuthology.orchestra.run.smithi069.stdout: 5 62027 19.50 MB/sec warmup 90 sec latency 19.452 ms 2024-04-22T23:12:41.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 62824 19.45 MB/sec warmup 91 sec latency 26.548 ms 2024-04-22T23:12:41.264 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:41 smithi069 ceph-mon[21859]: pgmap v284: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 10 MiB/s wr, 895 op/s 2024-04-22T23:12:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:41 smithi149 ceph-mon[23948]: pgmap v284: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 10 MiB/s wr, 895 op/s 2024-04-22T23:12:42.249 INFO:teuthology.orchestra.run.smithi069.stdout: 5 63290 19.45 MB/sec warmup 92 sec latency 41.691 ms 2024-04-22T23:12:43.251 INFO:teuthology.orchestra.run.smithi069.stdout: 5 63841 19.46 MB/sec warmup 93 sec latency 35.122 ms 2024-04-22T23:12:43.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:43 smithi069 ceph-mon[21859]: pgmap v285: 97 pgs: 97 active+clean; 313 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 878 op/s 2024-04-22T23:12:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:43 smithi149 ceph-mon[23948]: pgmap v285: 97 pgs: 97 active+clean; 313 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 878 op/s 2024-04-22T23:12:44.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 64340 19.42 MB/sec warmup 94 sec latency 42.085 ms 2024-04-22T23:12:44.615 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:44 smithi069 ceph-mon[21859]: pgmap v286: 97 pgs: 97 active+clean; 322 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 9.0 MiB/s wr, 821 op/s 2024-04-22T23:12:44.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:44 smithi149 ceph-mon[23948]: pgmap v286: 97 pgs: 97 active+clean; 322 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 9.0 MiB/s wr, 821 op/s 2024-04-22T23:12:45.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 64904 19.31 MB/sec warmup 95 sec latency 32.158 ms 2024-04-22T23:12:45.880 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:12:46.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 65358 19.21 MB/sec warmup 96 sec latency 100.729 ms 2024-04-22T23:12:46.773 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:12:46.773 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (6m) 98s ago 8m 24.4M - ba2b418f427c e0aa152f63bd 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (8m) 98s ago 8m 6400k - 17.2.0 e1d6a67b021e f29a69890fd2 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (7m) 101s ago 7m 6400k - 17.2.0 e1d6a67b021e d3576856bdfd 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (6m) 98s ago 7m 52.4M - 8.3.5 dad864ee21e9 c5880e24c661 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (2m) 98s ago 2m 13.4M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (2m) 101s ago 2m 12.1M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:9283 running (9m) 98s ago 9m 460M - 17.2.0 e1d6a67b021e 98c4390893c8 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283 running (7m) 101s ago 7m 413M - 17.2.0 e1d6a67b021e fa2d6964da01 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (9m) 98s ago 9m 48.3M 2048M 17.2.0 e1d6a67b021e 008ad8fe38c6 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (7m) 101s ago 7m 37.7M 2048M 17.2.0 e1d6a67b021e 1a87df28e650 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (109s) 98s ago 109s 49.9M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (104s) 101s ago 103s 50.7M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (7m) 98s ago 7m 18.9M - 1dbe0e931976 1175a40c25d7 2024-04-22T23:12:46.774 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (7m) 101s ago 7m 17.1M - 1dbe0e931976 fd4ddbc4e0f2 2024-04-22T23:12:46.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (6m) 98s ago 6m 45.4M 3216M 17.2.0 e1d6a67b021e 777db7a3d809 2024-04-22T23:12:46.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (6m) 98s ago 6m 42.5M 3216M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:12:46.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (5m) 98s ago 5m 43.0M 3216M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:12:46.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (5m) 98s ago 5m 42.5M 3216M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:12:46.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (4m) 101s ago 4m 39.8M 1166M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:12:46.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (4m) 101s ago 4m 42.0M 1166M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:12:46.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (4m) 101s ago 4m 39.8M 1166M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:12:46.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (3m) 101s ago 3m 41.1M 1166M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:12:46.775 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (6m) 98s ago 6m 62.4M - 514e6a882f6e b8ea50a9c418 2024-04-22T23:12:47.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 66035 19.18 MB/sec warmup 97 sec latency 107.063 ms 2024-04-22T23:12:47.309 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:12:47.309 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:12:47.309 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:12:47.310 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:12:47.379 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:47 smithi069 ceph-mon[21859]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:47.379 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:47 smithi069 ceph-mon[21859]: pgmap v287: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.8 MiB/s wr, 927 op/s 2024-04-22T23:12:47.380 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:47 smithi069 ceph-mon[21859]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:47 smithi149 ceph-mon[23948]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:47 smithi149 ceph-mon[23948]: pgmap v287: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.8 MiB/s wr, 927 op/s 2024-04-22T23:12:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:47 smithi149 ceph-mon[23948]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:47.729 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:12:47.729 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:12:47.729 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:12:47.729 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [], 2024-04-22T23:12:47.729 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "0/23 daemons upgraded", 2024-04-22T23:12:47.729 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c image on host smithi149" 2024-04-22T23:12:47.729 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:12:48.209 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:12:48.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:48 smithi069 ceph-mon[21859]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:48.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:48 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1003294389' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:12:48.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 66616 19.15 MB/sec warmup 98 sec latency 44.983 ms 2024-04-22T23:12:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:48 smithi149 ceph-mon[23948]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:48 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/1003294389' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:12:49.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:49 smithi069 ceph-mon[21859]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:49.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:49 smithi069 ceph-mon[21859]: pgmap v288: 97 pgs: 97 active+clean; 326 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.2 MiB/s wr, 955 op/s 2024-04-22T23:12:49.239 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:49 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/1423514578' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:12:49.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 67291 19.22 MB/sec warmup 99 sec latency 25.756 ms 2024-04-22T23:12:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:49 smithi149 ceph-mon[23948]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:12:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:49 smithi149 ceph-mon[23948]: pgmap v288: 97 pgs: 97 active+clean; 326 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.2 MiB/s wr, 955 op/s 2024-04-22T23:12:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:49 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/1423514578' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:12:50.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 67863 19.21 MB/sec warmup 100 sec latency 22.233 ms 2024-04-22T23:12:51.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 68413 19.10 MB/sec warmup 101 sec latency 43.666 ms 2024-04-22T23:12:51.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:51 smithi069 ceph-mon[21859]: pgmap v289: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 9.5 MiB/s wr, 876 op/s 2024-04-22T23:12:51.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:51 smithi149 ceph-mon[23948]: pgmap v289: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 9.5 MiB/s wr, 876 op/s 2024-04-22T23:12:52.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 68982 19.02 MB/sec warmup 102 sec latency 28.803 ms 2024-04-22T23:12:53.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 69581 18.99 MB/sec warmup 103 sec latency 39.573 ms 2024-04-22T23:12:53.283 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:53 smithi069 ceph-mon[21859]: pgmap v290: 97 pgs: 97 active+clean; 326 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.7 MiB/s wr, 879 op/s 2024-04-22T23:12:53.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:53 smithi149 ceph-mon[23948]: pgmap v290: 97 pgs: 97 active+clean; 326 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.7 MiB/s wr, 879 op/s 2024-04-22T23:12:54.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 70070 18.92 MB/sec warmup 104 sec latency 30.918 ms 2024-04-22T23:12:54.549 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:54 smithi069 ceph-mon[21859]: pgmap v291: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 936 op/s 2024-04-22T23:12:54.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:54 smithi149 ceph-mon[23948]: pgmap v291: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 936 op/s 2024-04-22T23:12:55.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 70596 18.96 MB/sec warmup 105 sec latency 29.482 ms 2024-04-22T23:12:56.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 71120 18.95 MB/sec warmup 106 sec latency 40.907 ms 2024-04-22T23:12:57.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 71548 18.87 MB/sec warmup 107 sec latency 54.898 ms 2024-04-22T23:12:57.337 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:57 smithi069 ceph-mon[21859]: pgmap v292: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 8.4 MiB/s wr, 893 op/s 2024-04-22T23:12:57.401 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:57 smithi149 ceph-mon[23948]: pgmap v292: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 8.4 MiB/s wr, 893 op/s 2024-04-22T23:12:58.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 72162 18.77 MB/sec warmup 108 sec latency 38.092 ms 2024-04-22T23:12:59.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 72909 18.77 MB/sec warmup 109 sec latency 55.958 ms 2024-04-22T23:12:59.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:12:59 smithi069 ceph-mon[21859]: pgmap v293: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 855 op/s 2024-04-22T23:12:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:12:59 smithi149 ceph-mon[23948]: pgmap v293: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 855 op/s 2024-04-22T23:13:00.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 73705 18.79 MB/sec warmup 110 sec latency 44.476 ms 2024-04-22T23:13:01.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 74276 18.84 MB/sec warmup 111 sec latency 24.039 ms 2024-04-22T23:13:01.382 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:01 smithi069 ceph-mon[21859]: pgmap v294: 97 pgs: 97 active+clean; 322 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 774 op/s 2024-04-22T23:13:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:01 smithi149 ceph-mon[23948]: pgmap v294: 97 pgs: 97 active+clean; 322 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 774 op/s 2024-04-22T23:13:02.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 74828 18.83 MB/sec warmup 112 sec latency 26.451 ms 2024-04-22T23:13:02.901 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:02 smithi069 ceph-mon[21859]: Upgrade: Updating mgr.smithi149.rxqkqk 2024-04-22T23:13:02.902 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:02 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:02.902 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:02 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.rxqkqk", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:13:02.902 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:02 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:13:02.902 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:02 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:13:02.902 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:02 smithi069 ceph-mon[21859]: Deploying daemon mgr.smithi149.rxqkqk on smithi149 2024-04-22T23:13:02.902 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:02 smithi069 ceph-mon[21859]: pgmap v295: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.0 MiB/s wr, 839 op/s 2024-04-22T23:13:02.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:02 smithi149 ceph-mon[23948]: Upgrade: Updating mgr.smithi149.rxqkqk 2024-04-22T23:13:02.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:02 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:02.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:02 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.rxqkqk", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:13:02.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:02 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:13:02.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:02 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:13:02.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:02 smithi149 ceph-mon[23948]: Deploying daemon mgr.smithi149.rxqkqk on smithi149 2024-04-22T23:13:02.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:02 smithi149 ceph-mon[23948]: pgmap v295: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.0 MiB/s wr, 839 op/s 2024-04-22T23:13:03.248 INFO:teuthology.orchestra.run.smithi069.stdout: 5 75187 18.72 MB/sec warmup 113 sec latency 27.908 ms 2024-04-22T23:13:04.249 INFO:teuthology.orchestra.run.smithi069.stdout: 5 75630 18.61 MB/sec warmup 114 sec latency 41.513 ms 2024-04-22T23:13:04.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:04 smithi069 ceph-mon[21859]: pgmap v296: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 807 op/s 2024-04-22T23:13:04.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:04 smithi149 ceph-mon[23948]: pgmap v296: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 807 op/s 2024-04-22T23:13:05.249 INFO:teuthology.orchestra.run.smithi069.stdout: 5 76267 18.60 MB/sec warmup 115 sec latency 41.934 ms 2024-04-22T23:13:06.249 INFO:teuthology.orchestra.run.smithi069.stdout: 5 77098 18.59 MB/sec warmup 116 sec latency 21.428 ms 2024-04-22T23:13:07.227 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:07 smithi069 ceph-mon[21859]: pgmap v297: 97 pgs: 97 active+clean; 333 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.8 MiB/s wr, 866 op/s 2024-04-22T23:13:07.249 INFO:teuthology.orchestra.run.smithi069.stdout: 5 77637 18.60 MB/sec warmup 117 sec latency 42.170 ms 2024-04-22T23:13:07.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:07 smithi149 ceph-mon[23948]: pgmap v297: 97 pgs: 97 active+clean; 333 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.8 MiB/s wr, 866 op/s 2024-04-22T23:13:08.249 INFO:teuthology.orchestra.run.smithi069.stdout: 5 78155 18.59 MB/sec warmup 118 sec latency 39.256 ms 2024-04-22T23:13:08.407 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:13:08.407 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:08 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:13:08.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:13:08.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:08 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:13:09.169 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:09 smithi069 ceph-mon[21859]: pgmap v298: 97 pgs: 97 active+clean; 336 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 8.8 MiB/s wr, 890 op/s 2024-04-22T23:13:09.252 INFO:teuthology.orchestra.run.smithi069.stdout: 5 78619 18.57 MB/sec warmup 119 sec latency 29.200 ms 2024-04-22T23:13:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:09 smithi149 ceph-mon[23948]: pgmap v298: 97 pgs: 97 active+clean; 336 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 8.8 MiB/s wr, 890 op/s 2024-04-22T23:13:11.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:11 smithi149 ceph-mon[23948]: pgmap v299: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 8.9 MiB/s wr, 774 op/s 2024-04-22T23:13:11.249 INFO:teuthology.orchestra.run.smithi069.stdout: 5 79497 8.36 MB/sec execute 1 sec latency 83.570 ms 2024-04-22T23:13:11.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:11 smithi069 ceph-mon[21859]: pgmap v299: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 8.9 MiB/s wr, 774 op/s 2024-04-22T23:13:12.249 INFO:teuthology.orchestra.run.smithi069.stdout: 5 80007 11.65 MB/sec execute 2 sec latency 84.968 ms 2024-04-22T23:13:13.249 INFO:teuthology.orchestra.run.smithi069.stdout: 5 80516 10.56 MB/sec execute 3 sec latency 66.659 ms 2024-04-22T23:13:13.290 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:13 smithi069 ceph-mon[21859]: pgmap v300: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 820 op/s 2024-04-22T23:13:13.291 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:13 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:13.291 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:13 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:13:13.291 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:13 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:13:13.291 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:13 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:13:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:13 smithi149 ceph-mon[23948]: pgmap v300: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 820 op/s 2024-04-22T23:13:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:13 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:13 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:13:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:13 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:13:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:13 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:13:14.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 81225 15.03 MB/sec execute 4 sec latency 22.872 ms 2024-04-22T23:13:14.604 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:14 smithi069 ceph-mon[21859]: pgmap v301: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.1 MiB/s wr, 807 op/s 2024-04-22T23:13:15.098 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:14 smithi149 ceph-mon[23948]: pgmap v301: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.1 MiB/s wr, 807 op/s 2024-04-22T23:13:15.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 81863 16.11 MB/sec execute 5 sec latency 68.454 ms 2024-04-22T23:13:16.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 82171 14.84 MB/sec execute 6 sec latency 84.823 ms 2024-04-22T23:13:17.169 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:17 smithi069 ceph-mon[21859]: pgmap v302: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 794 op/s 2024-04-22T23:13:17.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 82478 13.50 MB/sec execute 7 sec latency 49.865 ms 2024-04-22T23:13:17.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:17 smithi149 ceph-mon[23948]: pgmap v302: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 794 op/s 2024-04-22T23:13:18.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 82950 12.75 MB/sec execute 8 sec latency 68.753 ms 2024-04-22T23:13:18.644 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:13:19.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 83374 12.31 MB/sec execute 9 sec latency 40.399 ms 2024-04-22T23:13:19.452 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:19 smithi069 ceph-mon[21859]: pgmap v303: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.3 MiB/s wr, 670 op/s 2024-04-22T23:13:19.453 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:19.453 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:19 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:19.497 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (7m) 2m ago 8m 24.4M - ba2b418f427c e0aa152f63bd 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (8m) 2m ago 8m 6400k - 17.2.0 e1d6a67b021e f29a69890fd2 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (8m) 1s ago 8m 6400k - 17.2.0 e1d6a67b021e d3576856bdfd 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (7m) 2m ago 8m 52.4M - 8.3.5 dad864ee21e9 c5880e24c661 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (2m) 2m ago 2m 13.4M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (2m) 1s ago 2m 12.6M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:9283 running (9m) 2m ago 9m 460M - 17.2.0 e1d6a67b021e 98c4390893c8 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283 running (10s) 1s ago 7m 91.4M - 19.0.0-3252-g43be0201 a0434699c1d7 43be2fc88208 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (9m) 2m ago 10m 48.3M 2048M 17.2.0 e1d6a67b021e 008ad8fe38c6 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (7m) 1s ago 7m 38.1M 2048M 17.2.0 e1d6a67b021e 1a87df28e650 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (2m) 2m ago 2m 49.9M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (2m) 1s ago 2m 76.9M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:13:19.498 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (8m) 2m ago 8m 18.9M - 1dbe0e931976 1175a40c25d7 2024-04-22T23:13:19.499 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (7m) 1s ago 7m 17.2M - 1dbe0e931976 fd4ddbc4e0f2 2024-04-22T23:13:19.499 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (7m) 2m ago 7m 45.4M 3216M 17.2.0 e1d6a67b021e 777db7a3d809 2024-04-22T23:13:19.499 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (6m) 2m ago 6m 42.5M 3216M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:13:19.499 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (6m) 2m ago 6m 43.0M 3216M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:13:19.499 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (5m) 2m ago 5m 42.5M 3216M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:13:19.499 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (5m) 1s ago 5m 293M 1166M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:13:19.499 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (5m) 1s ago 4m 330M 1166M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:13:19.499 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (4m) 1s ago 4m 271M 1166M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:13:19.499 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (4m) 1s ago 4m 401M 1166M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:13:19.499 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (7m) 2m ago 7m 62.4M - 514e6a882f6e b8ea50a9c418 2024-04-22T23:13:19.615 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:19 smithi149 ceph-mon[23948]: pgmap v303: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.3 MiB/s wr, 670 op/s 2024-04-22T23:13:19.615 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:19.616 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:19 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:19.990 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:13:19.990 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:13:19.990 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:13:19.990 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:13:19.990 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:13:19.990 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:13:19.990 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:13:19.990 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:13:19.991 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T23:13:19.991 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:13:19.991 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:13:19.991 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:13:19.991 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:13:19.991 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:13:19.991 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-22T23:13:19.991 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:13:19.991 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:13:20.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 83889 12.51 MB/sec execute 10 sec latency 56.758 ms 2024-04-22T23:13:20.442 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:13:20.442 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:13:20.442 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:13:20.442 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [], 2024-04-22T23:13:20.442 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "1/23 daemons upgraded", 2024-04-22T23:13:20.442 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading mgr daemons" 2024-04-22T23:13:20.443 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:13:20.468 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:20 smithi069 ceph-mon[21859]: from='client.14692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:20.468 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:20 smithi069 ceph-mon[21859]: from='client.14696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:20.468 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:20 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/4084901035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:13:20.594 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:20 smithi149 ceph-mon[23948]: from='client.14692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:20.595 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:20 smithi149 ceph-mon[23948]: from='client.14696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:20.595 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:20 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/4084901035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:13:21.009 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:13:21.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 84428 12.73 MB/sec execute 11 sec latency 22.114 ms 2024-04-22T23:13:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:21 smithi149 ceph-mon[23948]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:21.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:21 smithi149 ceph-mon[23948]: pgmap v304: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 8.0 MiB/s wr, 765 op/s 2024-04-22T23:13:21.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:21 smithi149 ceph-mon[23948]: Standby manager daemon smithi149.rxqkqk restarted 2024-04-22T23:13:21.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:21 smithi149 ceph-mon[23948]: Standby manager daemon smithi149.rxqkqk started 2024-04-22T23:13:21.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:21 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/crt"}]: dispatch 2024-04-22T23:13:21.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:21 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:13:21.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:21 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/key"}]: dispatch 2024-04-22T23:13:21.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:21 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:13:21.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:21 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/564642623' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:13:21.477 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:21 smithi069 ceph-mon[21859]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:21.477 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:21 smithi069 ceph-mon[21859]: pgmap v304: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 8.0 MiB/s wr, 765 op/s 2024-04-22T23:13:21.477 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:21 smithi069 ceph-mon[21859]: Standby manager daemon smithi149.rxqkqk restarted 2024-04-22T23:13:21.477 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:21 smithi069 ceph-mon[21859]: Standby manager daemon smithi149.rxqkqk started 2024-04-22T23:13:21.477 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:21 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/crt"}]: dispatch 2024-04-22T23:13:21.478 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:21 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:13:21.478 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:21 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/key"}]: dispatch 2024-04-22T23:13:21.478 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:21 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:13:21.478 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:21 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/564642623' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:13:22.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 84853 13.07 MB/sec execute 12 sec latency 53.557 ms 2024-04-22T23:13:22.488 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:22 smithi069 ceph-mon[21859]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:22.488 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:22 smithi069 ceph-mon[21859]: mgrmap e22: smithi069.lyiiqg(active, since 8m), standbys: smithi149.rxqkqk 2024-04-22T23:13:22.694 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:22 smithi149 ceph-mon[23948]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:22.695 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:22 smithi149 ceph-mon[23948]: mgrmap e22: smithi069.lyiiqg(active, since 8m), standbys: smithi149.rxqkqk 2024-04-22T23:13:23.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 85400 13.55 MB/sec execute 13 sec latency 34.581 ms 2024-04-22T23:13:23.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:23 smithi069 ceph-mon[21859]: pgmap v305: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.1 MiB/s wr, 761 op/s 2024-04-22T23:13:23.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:23 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:23.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:23 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:23.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:23 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:23.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:23 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:13:23.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:23 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr fail", "who": "smithi069.lyiiqg"}]: dispatch 2024-04-22T23:13:23.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:23 smithi069 ceph-mon[21859]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T23:13:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:23 smithi149 ceph-mon[23948]: pgmap v305: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.1 MiB/s wr, 761 op/s 2024-04-22T23:13:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:23 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:23 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:23 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:13:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:23 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:13:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:23 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr fail", "who": "smithi069.lyiiqg"}]: dispatch 2024-04-22T23:13:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:23 smithi149 ceph-mon[23948]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T23:13:24.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 85807 13.25 MB/sec execute 14 sec latency 35.096 ms 2024-04-22T23:13:24.508 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: Upgrade: Need to upgrade myself (mgr.smithi069.lyiiqg) 2024-04-22T23:13:24.508 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: Failing over to other MGR 2024-04-22T23:13:24.508 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: Standby manager daemon smithi069.lyiiqg started 2024-04-22T23:13:24.508 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:13:24.508 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:13:24.508 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mgr metadata", "who": "smithi149.rxqkqk", "id": "smithi149.rxqkqk"}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mgr metadata", "who": "smithi069.lyiiqg", "id": "smithi069.lyiiqg"}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T23:13:24.509 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T23:13:24.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T23:13:24.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "mgr fail", "who": "smithi069.lyiiqg"}]': finished 2024-04-22T23:13:24.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: mgrmap e23: smithi149.rxqkqk(active, starting, since 1.00317s), standbys: smithi069.lyiiqg 2024-04-22T23:13:24.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: Manager daemon smithi149.rxqkqk is now available 2024-04-22T23:13:24.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:24.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:13:24.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:13:24.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.rxqkqk/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:13:24.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.rxqkqk/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:13:24.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.rxqkqk/trash_purge_schedule"}]: dispatch 2024-04-22T23:13:24.510 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:24 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.rxqkqk/trash_purge_schedule"}]: dispatch 2024-04-22T23:13:24.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: Upgrade: Need to upgrade myself (mgr.smithi069.lyiiqg) 2024-04-22T23:13:24.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: Failing over to other MGR 2024-04-22T23:13:24.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: Standby manager daemon smithi069.lyiiqg started 2024-04-22T23:13:24.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:13:24.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:13:24.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:13:24.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:13:24.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mgr metadata", "who": "smithi149.rxqkqk", "id": "smithi149.rxqkqk"}]: dispatch 2024-04-22T23:13:24.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mgr metadata", "who": "smithi069.lyiiqg", "id": "smithi069.lyiiqg"}]: dispatch 2024-04-22T23:13:24.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.14204 172.21.15.69:0/2625373259' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "mgr fail", "who": "smithi069.lyiiqg"}]': finished 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: mgrmap e23: smithi149.rxqkqk(active, starting, since 1.00317s), standbys: smithi069.lyiiqg 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: Manager daemon smithi149.rxqkqk is now available 2024-04-22T23:13:24.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:24.513 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:13:24.513 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:13:24.513 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.rxqkqk/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:13:24.513 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.rxqkqk/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:13:24.513 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.rxqkqk/trash_purge_schedule"}]: dispatch 2024-04-22T23:13:24.513 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:24 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi149.rxqkqk/trash_purge_schedule"}]: dispatch 2024-04-22T23:13:25.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 86323 13.01 MB/sec execute 15 sec latency 30.060 ms 2024-04-22T23:13:25.885 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:25 smithi069 ceph-mon[21859]: Deploying cephadm binary to smithi149 2024-04-22T23:13:25.885 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:25 smithi069 ceph-mon[21859]: Deploying cephadm binary to smithi069 2024-04-22T23:13:25.885 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:25 smithi069 ceph-mon[21859]: mgrmap e24: smithi149.rxqkqk(active, since 2s), standbys: smithi069.lyiiqg 2024-04-22T23:13:25.885 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:25 smithi069 ceph-mon[21859]: pgmap v3: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-22T23:13:25.885 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:25 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:25.885 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:25 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:25.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:25 smithi149 ceph-mon[23948]: Deploying cephadm binary to smithi149 2024-04-22T23:13:25.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:25 smithi149 ceph-mon[23948]: Deploying cephadm binary to smithi069 2024-04-22T23:13:25.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:25 smithi149 ceph-mon[23948]: mgrmap e24: smithi149.rxqkqk(active, since 2s), standbys: smithi069.lyiiqg 2024-04-22T23:13:25.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:25 smithi149 ceph-mon[23948]: pgmap v3: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-22T23:13:25.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:25 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:25.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:25 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:26.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 86788 12.77 MB/sec execute 16 sec latency 37.693 ms 2024-04-22T23:13:27.037 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:26 smithi069 ceph-mon[21859]: pgmap v4: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-22T23:13:27.037 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:26 smithi069 ceph-mon[21859]: mgrmap e25: smithi149.rxqkqk(active, since 3s), standbys: smithi069.lyiiqg 2024-04-22T23:13:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:26 smithi149 ceph-mon[23948]: pgmap v4: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-22T23:13:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:26 smithi149 ceph-mon[23948]: mgrmap e25: smithi149.rxqkqk(active, since 3s), standbys: smithi069.lyiiqg 2024-04-22T23:13:27.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 87302 12.88 MB/sec execute 17 sec latency 28.465 ms 2024-04-22T23:13:28.044 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:27 smithi069 ceph-mon[21859]: [22/Apr/2024:23:13:26] ENGINE Bus STARTING 2024-04-22T23:13:28.044 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:27 smithi069 ceph-mon[21859]: [22/Apr/2024:23:13:26] ENGINE Serving on https://172.21.15.149:7150 2024-04-22T23:13:28.045 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:27 smithi069 ceph-mon[21859]: [22/Apr/2024:23:13:26] ENGINE Serving on http://172.21.15.149:8765 2024-04-22T23:13:28.045 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:27 smithi069 ceph-mon[21859]: [22/Apr/2024:23:13:26] ENGINE Bus STARTED 2024-04-22T23:13:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:27 smithi149 ceph-mon[23948]: [22/Apr/2024:23:13:26] ENGINE Bus STARTING 2024-04-22T23:13:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:27 smithi149 ceph-mon[23948]: [22/Apr/2024:23:13:26] ENGINE Serving on https://172.21.15.149:7150 2024-04-22T23:13:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:27 smithi149 ceph-mon[23948]: [22/Apr/2024:23:13:26] ENGINE Serving on http://172.21.15.149:8765 2024-04-22T23:13:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:27 smithi149 ceph-mon[23948]: [22/Apr/2024:23:13:26] ENGINE Bus STARTED 2024-04-22T23:13:28.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 87962 13.05 MB/sec execute 18 sec latency 27.031 ms 2024-04-22T23:13:28.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:28 smithi069 ceph-mon[21859]: pgmap v5: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-22T23:13:28.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:28 smithi069 ceph-mon[21859]: mgrmap e26: smithi149.rxqkqk(active, since 5s), standbys: smithi069.lyiiqg 2024-04-22T23:13:29.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:28 smithi149 ceph-mon[23948]: pgmap v5: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-22T23:13:29.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:28 smithi149 ceph-mon[23948]: mgrmap e26: smithi149.rxqkqk(active, since 5s), standbys: smithi069.lyiiqg 2024-04-22T23:13:29.254 INFO:teuthology.orchestra.run.smithi069.stdout: 5 88468 13.38 MB/sec execute 19 sec latency 45.101 ms 2024-04-22T23:13:30.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 88975 13.64 MB/sec execute 20 sec latency 77.226 ms 2024-04-22T23:13:30.854 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:30 smithi069 ceph-mon[21859]: pgmap v6: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-22T23:13:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:30 smithi149 ceph-mon[23948]: pgmap v6: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-22T23:13:31.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 89307 13.38 MB/sec execute 21 sec latency 73.712 ms 2024-04-22T23:13:32.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 89662 13.03 MB/sec execute 22 sec latency 57.492 ms 2024-04-22T23:13:32.913 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:32 smithi069 ceph-mon[21859]: pgmap v7: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 5.6 MiB/s wr, 567 op/s 2024-04-22T23:13:32.948 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:32 smithi149 ceph-mon[23948]: pgmap v7: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 5.6 MiB/s wr, 567 op/s 2024-04-22T23:13:33.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 90088 12.79 MB/sec execute 23 sec latency 37.937 ms 2024-04-22T23:13:33.919 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:33 smithi069 ceph-mon[21859]: Standby manager daemon smithi069.lyiiqg restarted 2024-04-22T23:13:33.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:33 smithi069 ceph-mon[21859]: Standby manager daemon smithi069.lyiiqg started 2024-04-22T23:13:33.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:33 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.69:0/953588814' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.lyiiqg/crt"}]: dispatch 2024-04-22T23:13:33.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:33 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.69:0/953588814' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:13:33.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:33 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.69:0/953588814' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.lyiiqg/key"}]: dispatch 2024-04-22T23:13:33.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:33 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.69:0/953588814' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:13:33.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:33 smithi149 ceph-mon[23948]: Standby manager daemon smithi069.lyiiqg restarted 2024-04-22T23:13:33.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:33 smithi149 ceph-mon[23948]: Standby manager daemon smithi069.lyiiqg started 2024-04-22T23:13:33.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:33 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.69:0/953588814' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.lyiiqg/crt"}]: dispatch 2024-04-22T23:13:33.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:33 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.69:0/953588814' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:13:33.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:33 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.69:0/953588814' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.lyiiqg/key"}]: dispatch 2024-04-22T23:13:33.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:33 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.69:0/953588814' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:13:34.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 90510 12.63 MB/sec execute 24 sec latency 44.366 ms 2024-04-22T23:13:34.936 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:34 smithi069 ceph-mon[21859]: pgmap v8: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-04-22T23:13:34.936 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:34 smithi069 ceph-mon[21859]: mgrmap e27: smithi149.rxqkqk(active, since 11s), standbys: smithi069.lyiiqg 2024-04-22T23:13:34.936 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:34 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:34.937 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:34 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:34 smithi149 ceph-mon[23948]: pgmap v8: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-04-22T23:13:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:34 smithi149 ceph-mon[23948]: mgrmap e27: smithi149.rxqkqk(active, since 11s), standbys: smithi069.lyiiqg 2024-04-22T23:13:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:34 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:34 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:35.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 90976 12.65 MB/sec execute 25 sec latency 47.806 ms 2024-04-22T23:13:36.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 91418 12.65 MB/sec execute 26 sec latency 51.812 ms 2024-04-22T23:13:36.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:36 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:36.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:36 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:36.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:36 smithi149 ceph-mon[23948]: pgmap v9: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 6.1 MiB/s wr, 639 op/s 2024-04-22T23:13:36.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:36 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:36.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:36 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:36.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:36 smithi069 ceph-mon[21859]: pgmap v9: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 6.1 MiB/s wr, 639 op/s 2024-04-22T23:13:37.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 91775 12.67 MB/sec execute 27 sec latency 81.827 ms 2024-04-22T23:13:38.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 92299 12.89 MB/sec execute 28 sec latency 28.042 ms 2024-04-22T23:13:38.986 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:38 smithi069 ceph-mon[21859]: pgmap v10: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 5.6 MiB/s wr, 584 op/s 2024-04-22T23:13:38.986 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:38 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:38.986 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:38 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:38 smithi149 ceph-mon[23948]: pgmap v10: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 5.6 MiB/s wr, 584 op/s 2024-04-22T23:13:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:38 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:38 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:39.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 92759 12.92 MB/sec execute 29 sec latency 23.270 ms 2024-04-22T23:13:39.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:39 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:13:39.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:39 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:39.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:39 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:39.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:39 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:39.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:39 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:39.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:39 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd='[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]': finished 2024-04-22T23:13:39.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:39 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:13:39.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:39 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:39.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:39 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:39.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:39 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:39.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:39 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:39.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:39 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd='[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]': finished 2024-04-22T23:13:40.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 93148 12.74 MB/sec execute 30 sec latency 27.735 ms 2024-04-22T23:13:41.012 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: pgmap v11: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 5.6 MiB/s wr, 584 op/s 2024-04-22T23:13:41.012 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:41.012 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:41.012 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.012 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.012 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.012 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.012 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.012 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:41.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:13:41.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:40 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:13:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: pgmap v11: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 5.6 MiB/s wr, 584 op/s 2024-04-22T23:13:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:13:41.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:41.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:13:41.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:40 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:13:41.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 93656 12.58 MB/sec execute 31 sec latency 35.830 ms 2024-04-22T23:13:42.030 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: Adjusting osd_memory_target on smithi069 to 1936M 2024-04-22T23:13:42.030 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: Updating smithi069:/etc/ceph/ceph.conf 2024-04-22T23:13:42.030 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: Updating smithi149:/etc/ceph/ceph.conf 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: Updating smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: Updating smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: Updating smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.032 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.032 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:41 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: Adjusting osd_memory_target on smithi069 to 1936M 2024-04-22T23:13:42.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: Updating smithi069:/etc/ceph/ceph.conf 2024-04-22T23:13:42.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: Updating smithi149:/etc/ceph/ceph.conf 2024-04-22T23:13:42.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: Updating smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T23:13:42.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: Updating smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T23:13:42.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:13:42.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:13:42.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: Updating smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T23:13:42.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:41 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:42.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 94189 12.66 MB/sec execute 32 sec latency 29.818 ms 2024-04-22T23:13:43.039 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:42 smithi069 ceph-mon[21859]: Updating smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T23:13:43.039 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:42 smithi069 ceph-mon[21859]: Reconfiguring alertmanager.smithi069 (dependencies changed)... 2024-04-22T23:13:43.039 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:42 smithi069 ceph-mon[21859]: Reconfiguring daemon alertmanager.smithi069 on smithi069 2024-04-22T23:13:43.039 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:42 smithi069 ceph-mon[21859]: pgmap v12: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 8.5 MiB/s wr, 839 op/s 2024-04-22T23:13:43.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:42 smithi149 ceph-mon[23948]: Updating smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T23:13:43.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:42 smithi149 ceph-mon[23948]: Reconfiguring alertmanager.smithi069 (dependencies changed)... 2024-04-22T23:13:43.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:42 smithi149 ceph-mon[23948]: Reconfiguring daemon alertmanager.smithi069 on smithi069 2024-04-22T23:13:43.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:42 smithi149 ceph-mon[23948]: pgmap v12: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 8.5 MiB/s wr, 839 op/s 2024-04-22T23:13:43.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 94791 12.68 MB/sec execute 33 sec latency 35.356 ms 2024-04-22T23:13:44.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 95248 12.75 MB/sec execute 34 sec latency 49.334 ms 2024-04-22T23:13:44.557 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:44 smithi069 ceph-mon[21859]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.3 MiB/s wr, 510 op/s 2024-04-22T23:13:44.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:44 smithi149 ceph-mon[23948]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.3 MiB/s wr, 510 op/s 2024-04-22T23:13:45.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 95662 12.89 MB/sec execute 35 sec latency 26.561 ms 2024-04-22T23:13:46.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 96062 12.88 MB/sec execute 36 sec latency 43.234 ms 2024-04-22T23:13:46.863 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:46 smithi069 ceph-mon[21859]: pgmap v14: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 811 op/s 2024-04-22T23:13:47.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:46 smithi149 ceph-mon[23948]: pgmap v14: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 811 op/s 2024-04-22T23:13:47.255 INFO:teuthology.orchestra.run.smithi069.stdout: 5 96374 12.74 MB/sec execute 37 sec latency 37.366 ms 2024-04-22T23:13:48.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 96686 12.57 MB/sec execute 38 sec latency 43.334 ms 2024-04-22T23:13:48.891 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:48 smithi069 ceph-mon[21859]: pgmap v15: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 555 op/s 2024-04-22T23:13:49.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:48 smithi149 ceph-mon[23948]: pgmap v15: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 555 op/s 2024-04-22T23:13:49.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 97103 12.41 MB/sec execute 39 sec latency 41.895 ms 2024-04-22T23:13:50.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 97403 12.21 MB/sec execute 40 sec latency 49.886 ms 2024-04-22T23:13:50.929 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:50 smithi069 ceph-mon[21859]: pgmap v16: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 555 op/s 2024-04-22T23:13:50.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:50 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:50.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:50 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:50.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:50 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:50.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:50 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:50.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:50 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T23:13:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:50 smithi149 ceph-mon[23948]: pgmap v16: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 555 op/s 2024-04-22T23:13:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:50 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:50 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:50 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:50 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:50 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T23:13:51.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 97871 12.33 MB/sec execute 41 sec latency 45.706 ms 2024-04-22T23:13:51.547 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:13:51.939 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:51 smithi069 ceph-mon[21859]: Reconfiguring grafana.smithi069 (dependencies changed)... 2024-04-22T23:13:51.940 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:51 smithi069 ceph-mon[21859]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-22T23:13:51.940 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:51 smithi069 ceph-mon[21859]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T23:13:51.940 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:51 smithi069 ceph-mon[21859]: Reconfiguring daemon grafana.smithi069 on smithi069 2024-04-22T23:13:52.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:51 smithi149 ceph-mon[23948]: Reconfiguring grafana.smithi069 (dependencies changed)... 2024-04-22T23:13:52.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:51 smithi149 ceph-mon[23948]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-22T23:13:52.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:51 smithi149 ceph-mon[23948]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T23:13:52.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:51 smithi149 ceph-mon[23948]: Reconfiguring daemon grafana.smithi069 on smithi069 2024-04-22T23:13:52.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 98418 12.33 MB/sec execute 42 sec latency 53.092 ms 2024-04-22T23:13:52.515 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 starting - - - - 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (9m) 14s ago 9m 6400k - 17.2.0 e1d6a67b021e f29a69890fd2 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (8m) 18s ago 8m 6400k - 17.2.0 e1d6a67b021e d3576856bdfd 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (7m) 14s ago 8m 53.0M - 8.3.5 dad864ee21e9 c5880e24c661 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (3m) 14s ago 3m 36.5M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (3m) 18s ago 3m 13.1M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:9283 running (10m) 14s ago 10m 322M - 17.2.0 e1d6a67b021e 98c4390893c8 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283 running (43s) 18s ago 8m 495M - 19.0.0-3252-g43be0201 a0434699c1d7 43be2fc88208 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (10m) 14s ago 10m 54.9M 2048M 17.2.0 e1d6a67b021e 008ad8fe38c6 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (8m) 18s ago 8m 43.2M 2048M 17.2.0 e1d6a67b021e 1a87df28e650 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (2m) 14s ago 2m 248M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (2m) 18s ago 2m 77.7M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:13:52.516 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (8m) 14s ago 8m 18.8M - 1.3.1 1dbe0e931976 1175a40c25d7 2024-04-22T23:13:52.517 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (8m) 18s ago 8m 17.2M - 1.3.1 1dbe0e931976 fd4ddbc4e0f2 2024-04-22T23:13:52.517 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (7m) 14s ago 7m 352M 1936M 17.2.0 e1d6a67b021e 777db7a3d809 2024-04-22T23:13:52.517 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (7m) 14s ago 7m 276M 1936M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:13:52.517 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (6m) 14s ago 6m 221M 1936M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:13:52.517 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (6m) 14s ago 6m 305M 1936M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:13:52.517 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (5m) 18s ago 5m 322M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:13:52.517 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (5m) 18s ago 5m 356M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:13:52.517 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (5m) 18s ago 5m 287M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:13:52.517 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (4m) 18s ago 4m 429M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:13:52.517 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (8m) 14s ago 8m 67.0M - 2.33.4 514e6a882f6e b8ea50a9c418 2024-04-22T23:13:52.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:52 smithi069 ceph-mon[21859]: from='client.14734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:52.965 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:52 smithi069 ceph-mon[21859]: pgmap v17: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 785 op/s 2024-04-22T23:13:53.118 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:13:53.118 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:13:53.118 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 1 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 1 2024-04-22T23:13:53.119 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:13:53.120 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:13:53.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:52 smithi149 ceph-mon[23948]: from='client.14734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:53.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:52 smithi149 ceph-mon[23948]: pgmap v17: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 785 op/s 2024-04-22T23:13:53.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 98843 12.39 MB/sec execute 43 sec latency 40.267 ms 2024-04-22T23:13:53.663 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:13:53.663 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:13:53.663 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:13:53.663 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:13:53.663 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [], 2024-04-22T23:13:53.663 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "1/23 daemons upgraded", 2024-04-22T23:13:53.663 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "", 2024-04-22T23:13:53.663 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:13:53.663 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:13:53.977 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:53 smithi069 ceph-mon[21859]: from='client.14738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:53.977 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:53 smithi069 ceph-mon[21859]: from='client.14742 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:53.977 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:53 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2193159356' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:13:54.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:53 smithi149 ceph-mon[23948]: from='client.14738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:54.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:53 smithi149 ceph-mon[23948]: from='client.14742 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:54.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:53 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2193159356' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:13:54.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 99272 12.44 MB/sec execute 44 sec latency 34.195 ms 2024-04-22T23:13:54.306 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:13:54.982 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:54 smithi069 ceph-mon[21859]: from='client.24477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:54.982 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:54 smithi069 ceph-mon[21859]: pgmap v18: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 530 op/s 2024-04-22T23:13:54.982 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:54 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:13:54.982 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:54 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3311327814' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:13:55.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:54 smithi149 ceph-mon[23948]: from='client.24477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:13:55.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:54 smithi149 ceph-mon[23948]: pgmap v18: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 530 op/s 2024-04-22T23:13:55.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:54 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:13:55.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:54 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/3311327814' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:13:55.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 99722 12.47 MB/sec execute 45 sec latency 27.332 ms 2024-04-22T23:13:56.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 100037 12.42 MB/sec execute 46 sec latency 48.080 ms 2024-04-22T23:13:57.006 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:56 smithi069 ceph-mon[21859]: pgmap v19: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 790 op/s 2024-04-22T23:13:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:56 smithi149 ceph-mon[23948]: pgmap v19: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 790 op/s 2024-04-22T23:13:57.257 INFO:teuthology.orchestra.run.smithi069.stdout: 5 100374 12.21 MB/sec execute 47 sec latency 41.271 ms 2024-04-22T23:13:58.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 100763 12.11 MB/sec execute 48 sec latency 56.636 ms 2024-04-22T23:13:58.561 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:58 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:58.561 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:58 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:58.562 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:58 smithi069 ceph-mon[21859]: Reconfiguring node-exporter.smithi069 (dependencies changed)... 2024-04-22T23:13:58.562 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:58 smithi069 ceph-mon[21859]: Reconfiguring daemon node-exporter.smithi069 on smithi069 2024-04-22T23:13:58.562 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:13:58 smithi069 ceph-mon[21859]: pgmap v20: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 489 op/s 2024-04-22T23:13:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:58 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:58 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:13:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:58 smithi149 ceph-mon[23948]: Reconfiguring node-exporter.smithi069 (dependencies changed)... 2024-04-22T23:13:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:58 smithi149 ceph-mon[23948]: Reconfiguring daemon node-exporter.smithi069 on smithi069 2024-04-22T23:13:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:13:58 smithi149 ceph-mon[23948]: pgmap v20: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 489 op/s 2024-04-22T23:13:59.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 101177 12.10 MB/sec execute 49 sec latency 36.741 ms 2024-04-22T23:14:00.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 101630 12.07 MB/sec execute 50 sec latency 42.413 ms 2024-04-22T23:14:00.867 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:00 smithi069 ceph-mon[21859]: pgmap v21: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 489 op/s 2024-04-22T23:14:00.868 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:00 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:00.868 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:00 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:01.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:00 smithi149 ceph-mon[23948]: pgmap v21: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 489 op/s 2024-04-22T23:14:01.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:00 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:01.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:00 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:01.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 102109 12.09 MB/sec execute 51 sec latency 38.015 ms 2024-04-22T23:14:01.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:01 smithi069 ceph-mon[21859]: Reconfiguring prometheus.smithi069 (dependencies changed)... 2024-04-22T23:14:01.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:01 smithi069 ceph-mon[21859]: Reconfiguring daemon prometheus.smithi069 on smithi069 2024-04-22T23:14:02.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:01 smithi149 ceph-mon[23948]: Reconfiguring prometheus.smithi069 (dependencies changed)... 2024-04-22T23:14:02.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:01 smithi149 ceph-mon[23948]: Reconfiguring daemon prometheus.smithi069 on smithi069 2024-04-22T23:14:02.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 102467 12.13 MB/sec execute 52 sec latency 43.925 ms 2024-04-22T23:14:02.912 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:02 smithi069 ceph-mon[21859]: pgmap v22: 97 pgs: 97 active+clean; 325 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 736 op/s 2024-04-22T23:14:03.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:02 smithi149 ceph-mon[23948]: pgmap v22: 97 pgs: 97 active+clean; 325 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 736 op/s 2024-04-22T23:14:03.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 102881 12.20 MB/sec execute 53 sec latency 41.923 ms 2024-04-22T23:14:04.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 103263 12.20 MB/sec execute 54 sec latency 46.252 ms 2024-04-22T23:14:04.684 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:04 smithi069 ceph-mon[21859]: pgmap v23: 97 pgs: 97 active+clean; 325 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 506 op/s 2024-04-22T23:14:04.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:04 smithi149 ceph-mon[23948]: pgmap v23: 97 pgs: 97 active+clean; 325 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 506 op/s 2024-04-22T23:14:05.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 103528 12.07 MB/sec execute 55 sec latency 47.551 ms 2024-04-22T23:14:06.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 103800 11.95 MB/sec execute 56 sec latency 55.134 ms 2024-04-22T23:14:06.981 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:06 smithi069 ceph-mon[21859]: pgmap v24: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 737 op/s 2024-04-22T23:14:07.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:06 smithi149 ceph-mon[23948]: pgmap v24: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 737 op/s 2024-04-22T23:14:07.256 INFO:teuthology.orchestra.run.smithi069.stdout: 5 104184 11.84 MB/sec execute 57 sec latency 68.401 ms 2024-04-22T23:14:08.257 INFO:teuthology.orchestra.run.smithi069.stdout: 5 104538 11.76 MB/sec execute 58 sec latency 42.268 ms 2024-04-22T23:14:09.032 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:08 smithi069 ceph-mon[21859]: pgmap v25: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 476 op/s 2024-04-22T23:14:09.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:08 smithi149 ceph-mon[23948]: pgmap v25: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 476 op/s 2024-04-22T23:14:09.257 INFO:teuthology.orchestra.run.smithi069.stdout: 5 104922 11.75 MB/sec execute 59 sec latency 33.580 ms 2024-04-22T23:14:10.058 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:09 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:14:10.058 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:09 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:10.059 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:09 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:10.150 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:09 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:14:10.151 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:09 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:10.151 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:09 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:10.257 INFO:teuthology.orchestra.run.smithi069.stdout: 5 105436 11.78 MB/sec execute 60 sec latency 56.794 ms 2024-04-22T23:14:11.064 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:10 smithi149 ceph-mon[23948]: Reconfiguring node-exporter.smithi149 (dependencies changed)... 2024-04-22T23:14:11.064 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:10 smithi149 ceph-mon[23948]: Reconfiguring daemon node-exporter.smithi149 on smithi149 2024-04-22T23:14:11.064 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:10 smithi149 ceph-mon[23948]: pgmap v26: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 476 op/s 2024-04-22T23:14:11.072 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:10 smithi069 ceph-mon[21859]: Reconfiguring node-exporter.smithi149 (dependencies changed)... 2024-04-22T23:14:11.072 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:10 smithi069 ceph-mon[21859]: Reconfiguring daemon node-exporter.smithi149 on smithi149 2024-04-22T23:14:11.072 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:10 smithi069 ceph-mon[21859]: pgmap v26: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 476 op/s 2024-04-22T23:14:11.257 INFO:teuthology.orchestra.run.smithi069.stdout: 5 105963 11.88 MB/sec execute 61 sec latency 26.025 ms 2024-04-22T23:14:12.257 INFO:teuthology.orchestra.run.smithi069.stdout: 5 106341 11.95 MB/sec execute 62 sec latency 54.880 ms 2024-04-22T23:14:12.346 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:12.346 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:12.346 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T23:14:12.346 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T23:14:12.346 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi069.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T23:14:12.346 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi069.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T23:14:12.346 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:12.346 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T23:14:12.346 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T23:14:12.347 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi069.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T23:14:12.347 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi069.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T23:14:12.347 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:12.347 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:14:12.347 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:14:12.347 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi069.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T23:14:12.347 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi069.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T23:14:12.347 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:12.347 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:14:12.347 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: Upgrade: Need to upgrade myself (mgr.smithi149.rxqkqk) 2024-04-22T23:14:12.347 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: Upgrade: Need to upgrade myself (mgr.smithi149.rxqkqk) 2024-04-22T23:14:12.347 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:12 smithi069 ceph-mon[21859]: pgmap v27: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 699 op/s 2024-04-22T23:14:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T23:14:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T23:14:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi069.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T23:14:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi069.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T23:14:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi069.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi069.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi069.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi069.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: Upgrade: Need to upgrade myself (mgr.smithi149.rxqkqk) 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: Upgrade: Need to upgrade myself (mgr.smithi149.rxqkqk) 2024-04-22T23:14:12.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:12 smithi149 ceph-mon[23948]: pgmap v27: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 699 op/s 2024-04-22T23:14:13.257 INFO:teuthology.orchestra.run.smithi069.stdout: 5 106760 11.92 MB/sec execute 63 sec latency 29.990 ms 2024-04-22T23:14:14.257 INFO:teuthology.orchestra.run.smithi069.stdout: 5 107065 11.86 MB/sec execute 64 sec latency 37.470 ms 2024-04-22T23:14:14.904 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:14 smithi069 ceph-mon[21859]: pgmap v28: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 452 op/s 2024-04-22T23:14:14.904 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:14 smithi069 ceph-mon[21859]: Upgrade: Updating mgr.smithi069.lyiiqg 2024-04-22T23:14:14.905 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:14 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:14.905 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:14 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.lyiiqg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:14:14.905 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:14 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.lyiiqg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:14:14.905 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:14 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:14:14.905 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:14 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:14:14.905 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:14 smithi069 ceph-mon[21859]: Deploying daemon mgr.smithi069.lyiiqg on smithi069 2024-04-22T23:14:15.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:14 smithi149 ceph-mon[23948]: pgmap v28: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 452 op/s 2024-04-22T23:14:15.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:14 smithi149 ceph-mon[23948]: Upgrade: Updating mgr.smithi069.lyiiqg 2024-04-22T23:14:15.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:14 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:15.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:14 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.lyiiqg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:14:15.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:14 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.lyiiqg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:14:15.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:14 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:14:15.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:14 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:14:15.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:14 smithi149 ceph-mon[23948]: Deploying daemon mgr.smithi069.lyiiqg on smithi069 2024-04-22T23:14:15.257 INFO:teuthology.orchestra.run.smithi069.stdout: 5 107382 11.77 MB/sec execute 65 sec latency 63.124 ms 2024-04-22T23:14:16.263 INFO:teuthology.orchestra.run.smithi069.stdout: 5 107797 11.69 MB/sec execute 66 sec latency 44.966 ms 2024-04-22T23:14:16.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:16 smithi069 ceph-mon[21859]: pgmap v29: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.5 MiB/s wr, 700 op/s 2024-04-22T23:14:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:16 smithi149 ceph-mon[23948]: pgmap v29: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.5 MiB/s wr, 700 op/s 2024-04-22T23:14:17.266 INFO:teuthology.orchestra.run.smithi069.stdout: 5 108120 11.63 MB/sec execute 67 sec latency 57.391 ms 2024-04-22T23:14:18.266 INFO:teuthology.orchestra.run.smithi069.stdout: 5 108572 11.67 MB/sec execute 68 sec latency 39.194 ms 2024-04-22T23:14:18.993 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:18 smithi069 ceph-mon[21859]: pgmap v30: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 469 op/s 2024-04-22T23:14:19.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:18 smithi149 ceph-mon[23948]: pgmap v30: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 469 op/s 2024-04-22T23:14:19.266 INFO:teuthology.orchestra.run.smithi069.stdout: 5 109254 11.72 MB/sec execute 69 sec latency 38.815 ms 2024-04-22T23:14:20.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 109772 11.87 MB/sec execute 70 sec latency 42.342 ms 2024-04-22T23:14:21.008 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:20 smithi069 ceph-mon[21859]: pgmap v31: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 469 op/s 2024-04-22T23:14:21.008 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:20 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:21.008 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:20 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:21.008 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:20 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:14:21.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:20 smithi149 ceph-mon[23948]: pgmap v31: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 469 op/s 2024-04-22T23:14:21.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:20 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:21.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:20 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:21.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:20 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:14:21.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 110311 11.89 MB/sec execute 71 sec latency 39.444 ms 2024-04-22T23:14:22.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 110783 11.93 MB/sec execute 72 sec latency 30.662 ms 2024-04-22T23:14:22.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:22 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:22.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:22 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:22.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:22 smithi069 ceph-mon[21859]: pgmap v32: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 740 op/s 2024-04-22T23:14:22.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:22 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:22.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:22 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:22.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:22 smithi149 ceph-mon[23948]: pgmap v32: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 740 op/s 2024-04-22T23:14:23.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 111249 11.86 MB/sec execute 73 sec latency 43.628 ms 2024-04-22T23:14:24.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 111676 11.81 MB/sec execute 74 sec latency 34.941 ms 2024-04-22T23:14:24.630 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:24 smithi069 ceph-mon[21859]: pgmap v33: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 518 op/s 2024-04-22T23:14:24.630 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:24 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:14:24.858 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:24 smithi149 ceph-mon[23948]: pgmap v33: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 518 op/s 2024-04-22T23:14:24.858 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:24 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:14:24.907 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:14:25.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 112038 11.80 MB/sec execute 75 sec latency 59.804 ms 2024-04-22T23:14:25.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:25 smithi069 ceph-mon[21859]: from='client.14762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:14:25.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:25 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:25.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:25 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:25.806 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:14:25.807 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 starting - - - - 2024-04-22T23:14:25.807 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (9m) 47s ago 9m 6400k - 17.2.0 e1d6a67b021e f29a69890fd2 2024-04-22T23:14:25.807 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (9m) 0s ago 9m 6400k - 17.2.0 e1d6a67b021e d3576856bdfd 2024-04-22T23:14:25.807 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 starting - - - - 2024-04-22T23:14:25.807 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (3m) 47s ago 3m 36.5M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:14:25.807 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (3m) 0s ago 3m 13.3M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:14:25.807 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 starting - - - - 2024-04-22T23:14:25.807 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283 running (76s) 0s ago 9m 509M - 19.0.0-3252-g43be0201 a0434699c1d7 43be2fc88208 2024-04-22T23:14:25.807 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (11m) 47s ago 11m 54.9M 2048M 17.2.0 e1d6a67b021e 008ad8fe38c6 2024-04-22T23:14:25.807 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (8m) 0s ago 8m 43.0M 2048M 17.2.0 e1d6a67b021e 1a87df28e650 2024-04-22T23:14:25.808 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (3m) 47s ago 3m 248M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:14:25.808 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (3m) 0s ago 3m 81.6M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:14:25.808 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 starting - - - - 2024-04-22T23:14:25.808 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (15s) 0s ago 8m 8938k - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:14:25.808 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (8m) 47s ago 8m 352M 1936M 17.2.0 e1d6a67b021e 777db7a3d809 2024-04-22T23:14:25.808 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (7m) 47s ago 7m 276M 1936M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:14:25.809 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (7m) 47s ago 7m 221M 1936M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:14:25.809 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (6m) 47s ago 6m 305M 1936M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:14:25.809 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (6m) 0s ago 6m 370M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:14:25.809 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (6m) 0s ago 6m 422M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:14:25.809 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (5m) 0s ago 5m 348M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:14:25.809 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (5m) 0s ago 5m 493M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:14:25.809 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 starting - - - - 2024-04-22T23:14:25.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:25 smithi149 ceph-mon[23948]: from='client.14762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:14:25.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:25 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:25.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:25 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:26.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 112377 11.79 MB/sec execute 76 sec latency 94.375 ms 2024-04-22T23:14:26.319 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:14:26.319 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:14:26.319 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:14:26.319 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:14:26.319 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:14:26.319 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T23:14:26.319 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 1 2024-04-22T23:14:26.320 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:14:26.320 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:14:26.320 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T23:14:26.320 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:14:26.320 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:14:26.320 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:14:26.320 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:14:26.320 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:14:26.320 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T23:14:26.320 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 1 2024-04-22T23:14:26.320 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:14:26.320 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:14:26.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:26 smithi069 ceph-mon[21859]: from='client.24491 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:14:26.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:26 smithi069 ceph-mon[21859]: pgmap v34: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 806 op/s 2024-04-22T23:14:26.662 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:26 smithi069 ceph-mon[21859]: from='client.14770 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:14:26.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:26 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/4048787373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:14:26.808 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:14:26.808 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:14:26.808 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:14:26.808 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:14:26.808 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [], 2024-04-22T23:14:26.808 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "", 2024-04-22T23:14:26.808 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading mgr daemons", 2024-04-22T23:14:26.808 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:14:26.808 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:14:26.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:26 smithi149 ceph-mon[23948]: from='client.24491 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:14:26.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:26 smithi149 ceph-mon[23948]: pgmap v34: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 806 op/s 2024-04-22T23:14:26.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:26 smithi149 ceph-mon[23948]: from='client.14770 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:14:26.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:26 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/4048787373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:14:27.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 112856 11.79 MB/sec execute 77 sec latency 35.369 ms 2024-04-22T23:14:27.491 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:14:27.685 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:27 smithi069 ceph-mon[21859]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:14:27.685 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:27 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/3495997564' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:14:27.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:27 smithi149 ceph-mon[23948]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:14:27.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:27 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/3495997564' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:14:28.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 113135 11.76 MB/sec execute 78 sec latency 33.469 ms 2024-04-22T23:14:28.690 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:28 smithi069 ceph-mon[21859]: pgmap v35: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 559 op/s 2024-04-22T23:14:28.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:28 smithi149 ceph-mon[23948]: pgmap v35: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 559 op/s 2024-04-22T23:14:29.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 113483 11.81 MB/sec execute 79 sec latency 41.450 ms 2024-04-22T23:14:30.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 113832 11.74 MB/sec execute 80 sec latency 48.065 ms 2024-04-22T23:14:30.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:30 smithi069 ceph-mon[21859]: pgmap v36: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 559 op/s 2024-04-22T23:14:30.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:30 smithi069 ceph-mon[21859]: Standby manager daemon smithi069.lyiiqg restarted 2024-04-22T23:14:30.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:30 smithi069 ceph-mon[21859]: Standby manager daemon smithi069.lyiiqg started 2024-04-22T23:14:30.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:30 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.lyiiqg/crt"}]: dispatch 2024-04-22T23:14:30.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:30 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:14:30.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:30 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.lyiiqg/key"}]: dispatch 2024-04-22T23:14:30.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:30 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:14:30.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:30 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:30.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:30 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:30 smithi149 ceph-mon[23948]: pgmap v36: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 559 op/s 2024-04-22T23:14:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:30 smithi149 ceph-mon[23948]: Standby manager daemon smithi069.lyiiqg restarted 2024-04-22T23:14:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:30 smithi149 ceph-mon[23948]: Standby manager daemon smithi069.lyiiqg started 2024-04-22T23:14:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:30 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.lyiiqg/crt"}]: dispatch 2024-04-22T23:14:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:30 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:14:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:30 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.lyiiqg/key"}]: dispatch 2024-04-22T23:14:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:30 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:14:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:30 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:30 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:31.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 114114 11.72 MB/sec execute 81 sec latency 47.720 ms 2024-04-22T23:14:31.972 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:31 smithi069 ceph-mon[21859]: mgrmap e28: smithi149.rxqkqk(active, since 68s), standbys: smithi069.lyiiqg 2024-04-22T23:14:32.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:31 smithi149 ceph-mon[23948]: mgrmap e28: smithi149.rxqkqk(active, since 68s), standbys: smithi069.lyiiqg 2024-04-22T23:14:32.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 114433 11.66 MB/sec execute 82 sec latency 52.887 ms 2024-04-22T23:14:32.986 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:32 smithi069 ceph-mon[21859]: pgmap v37: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 766 op/s 2024-04-22T23:14:32.986 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:32 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:32.986 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:32 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:32 smithi149 ceph-mon[23948]: pgmap v37: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 766 op/s 2024-04-22T23:14:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:32 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:32 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:33.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 114858 11.59 MB/sec execute 83 sec latency 52.382 ms 2024-04-22T23:14:34.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 115249 11.56 MB/sec execute 84 sec latency 35.239 ms 2024-04-22T23:14:34.758 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:34 smithi069 ceph-mon[21859]: pgmap v38: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 494 op/s 2024-04-22T23:14:34.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:34 smithi149 ceph-mon[23948]: pgmap v38: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 494 op/s 2024-04-22T23:14:35.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 115792 11.64 MB/sec execute 85 sec latency 29.937 ms 2024-04-22T23:14:36.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 116275 11.62 MB/sec execute 86 sec latency 42.753 ms 2024-04-22T23:14:37.032 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:36 smithi069 ceph-mon[21859]: pgmap v39: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 732 op/s 2024-04-22T23:14:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:36 smithi149 ceph-mon[23948]: pgmap v39: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 732 op/s 2024-04-22T23:14:37.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 116685 11.65 MB/sec execute 87 sec latency 35.734 ms 2024-04-22T23:14:38.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 117272 11.74 MB/sec execute 88 sec latency 32.895 ms 2024-04-22T23:14:39.052 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:38 smithi069 ceph-mon[21859]: pgmap v40: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 444 op/s 2024-04-22T23:14:39.052 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:38 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:39.052 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:38 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:39.052 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:38 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:14:39.052 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:38 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:14:39.052 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:38 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:39.052 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:38 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:39.053 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:38 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:14:39.053 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:38 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mgr fail", "who": "smithi149.rxqkqk"}]: dispatch 2024-04-22T23:14:39.053 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:38 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mgr fail", "who": "smithi149.rxqkqk"}]: dispatch 2024-04-22T23:14:39.053 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:38 smithi069 ceph-mon[21859]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T23:14:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:38 smithi149 ceph-mon[23948]: pgmap v40: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 444 op/s 2024-04-22T23:14:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:38 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:38 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:38 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:14:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:38 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:14:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:38 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:38 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' 2024-04-22T23:14:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:38 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:14:39.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:38 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mgr fail", "who": "smithi149.rxqkqk"}]: dispatch 2024-04-22T23:14:39.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:38 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "mgr fail", "who": "smithi149.rxqkqk"}]: dispatch 2024-04-22T23:14:39.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:38 smithi149 ceph-mon[23948]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T23:14:39.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 117796 11.77 MB/sec execute 89 sec latency 33.131 ms 2024-04-22T23:14:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: Upgrade: Need to upgrade myself (mgr.smithi149.rxqkqk) 2024-04-22T23:14:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: Upgrade: Need to upgrade myself (mgr.smithi149.rxqkqk) 2024-04-22T23:14:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: Failing over to other MGR 2024-04-22T23:14:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:14:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: Standby manager daemon smithi149.rxqkqk started 2024-04-22T23:14:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd='[{"prefix": "mgr fail", "who": "smithi149.rxqkqk"}]': finished 2024-04-22T23:14:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: mgrmap e29: smithi069.lyiiqg(active, starting, since 0.858691s), standbys: smithi149.rxqkqk 2024-04-22T23:14:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:14:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:14:39.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi069.lyiiqg", "id": "smithi069.lyiiqg"}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi149.rxqkqk", "id": "smithi149.rxqkqk"}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T23:14:39.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T23:14:39.953 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T23:14:39.953 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: Manager daemon smithi069.lyiiqg is now available 2024-04-22T23:14:39.953 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:14:39.953 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:14:39.953 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:14:39.953 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:39 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:14:40.057 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: Upgrade: Need to upgrade myself (mgr.smithi149.rxqkqk) 2024-04-22T23:14:40.057 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: Upgrade: Need to upgrade myself (mgr.smithi149.rxqkqk) 2024-04-22T23:14:40.057 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: Failing over to other MGR 2024-04-22T23:14:40.058 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.24407 172.21.15.149:0/1206588483' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:14:40.058 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: Standby manager daemon smithi149.rxqkqk started 2024-04-22T23:14:40.058 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.24407 ' entity='mgr.smithi149.rxqkqk' cmd='[{"prefix": "mgr fail", "who": "smithi149.rxqkqk"}]': finished 2024-04-22T23:14:40.058 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: mgrmap e29: smithi069.lyiiqg(active, starting, since 0.858691s), standbys: smithi149.rxqkqk 2024-04-22T23:14:40.058 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:14:40.058 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:14:40.059 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:14:40.059 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:14:40.059 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi069.lyiiqg", "id": "smithi069.lyiiqg"}]: dispatch 2024-04-22T23:14:40.059 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi149.rxqkqk", "id": "smithi149.rxqkqk"}]: dispatch 2024-04-22T23:14:40.059 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:14:40.059 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:14:40.060 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:14:40.060 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:14:40.060 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:14:40.060 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:14:40.060 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:14:40.060 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:14:40.061 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T23:14:40.061 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T23:14:40.061 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T23:14:40.061 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: Manager daemon smithi069.lyiiqg is now available 2024-04-22T23:14:40.061 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:14:40.061 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:14:40.061 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:14:40.062 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:39 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:14:40.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 118391 11.75 MB/sec execute 90 sec latency 27.137 ms 2024-04-22T23:14:41.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 118913 11.73 MB/sec execute 91 sec latency 27.831 ms 2024-04-22T23:14:41.312 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:41 smithi069 ceph-mon[21859]: mgrmap e30: smithi069.lyiiqg(active, since 1.86471s), standbys: smithi149.rxqkqk 2024-04-22T23:14:41.312 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:41 smithi069 ceph-mon[21859]: [22/Apr/2024:23:14:40] ENGINE Bus STARTING 2024-04-22T23:14:41.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:41 smithi149 ceph-mon[23948]: mgrmap e30: smithi069.lyiiqg(active, since 1.86471s), standbys: smithi149.rxqkqk 2024-04-22T23:14:41.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:41 smithi149 ceph-mon[23948]: [22/Apr/2024:23:14:40] ENGINE Bus STARTING 2024-04-22T23:14:42.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 119630 11.83 MB/sec execute 92 sec latency 28.031 ms 2024-04-22T23:14:42.321 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:42 smithi069 ceph-mon[21859]: [22/Apr/2024:23:14:41] ENGINE Serving on https://172.21.15.69:7150 2024-04-22T23:14:42.321 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:42 smithi069 ceph-mon[21859]: [22/Apr/2024:23:14:41] ENGINE Serving on http://172.21.15.69:8765 2024-04-22T23:14:42.321 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:42 smithi069 ceph-mon[21859]: [22/Apr/2024:23:14:41] ENGINE Bus STARTED 2024-04-22T23:14:42.322 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:42 smithi069 ceph-mon[21859]: pgmap v4: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-04-22T23:14:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:42 smithi149 ceph-mon[23948]: [22/Apr/2024:23:14:41] ENGINE Serving on https://172.21.15.69:7150 2024-04-22T23:14:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:42 smithi149 ceph-mon[23948]: [22/Apr/2024:23:14:41] ENGINE Serving on http://172.21.15.69:8765 2024-04-22T23:14:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:42 smithi149 ceph-mon[23948]: [22/Apr/2024:23:14:41] ENGINE Bus STARTED 2024-04-22T23:14:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:42 smithi149 ceph-mon[23948]: pgmap v4: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-04-22T23:14:43.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 120141 11.90 MB/sec execute 93 sec latency 39.158 ms 2024-04-22T23:14:43.324 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:43 smithi149 ceph-mon[23948]: mgrmap e31: smithi069.lyiiqg(active, since 3s), standbys: smithi149.rxqkqk 2024-04-22T23:14:43.336 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:43 smithi069 ceph-mon[21859]: mgrmap e31: smithi069.lyiiqg(active, since 3s), standbys: smithi149.rxqkqk 2024-04-22T23:14:44.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 120526 11.95 MB/sec execute 94 sec latency 36.589 ms 2024-04-22T23:14:44.564 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:44 smithi069 ceph-mon[21859]: pgmap v5: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-04-22T23:14:44.565 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:44 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:44.565 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:44 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:44.845 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:44 smithi149 ceph-mon[23948]: pgmap v5: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-04-22T23:14:44.845 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:44 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:44.845 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:44 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:45.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 120847 11.89 MB/sec execute 95 sec latency 63.128 ms 2024-04-22T23:14:46.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 121266 11.90 MB/sec execute 96 sec latency 50.153 ms 2024-04-22T23:14:46.637 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:46 smithi069 ceph-mon[21859]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 2.2 MiB/s wr, 105 op/s 2024-04-22T23:14:46.638 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:46 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:46.638 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:46 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:46.638 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:46 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:14:46.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:46 smithi149 ceph-mon[23948]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 2.2 MiB/s wr, 105 op/s 2024-04-22T23:14:46.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:46 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:46.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:46 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:46.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:46 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:14:47.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 122131 11.93 MB/sec execute 97 sec latency 21.456 ms 2024-04-22T23:14:48.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 122906 12.03 MB/sec execute 98 sec latency 23.625 ms 2024-04-22T23:14:48.672 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:48 smithi069 ceph-mon[21859]: pgmap v7: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 5.5 MiB/s wr, 514 op/s 2024-04-22T23:14:48.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:48 smithi069 ceph-mon[21859]: Standby manager daemon smithi149.rxqkqk restarted 2024-04-22T23:14:48.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:48 smithi069 ceph-mon[21859]: Standby manager daemon smithi149.rxqkqk started 2024-04-22T23:14:48.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:48 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/2401231385' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/crt"}]: dispatch 2024-04-22T23:14:48.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:48 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/2401231385' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:14:48.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:48 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/2401231385' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/key"}]: dispatch 2024-04-22T23:14:48.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:48 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/2401231385' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:14:48.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:48 smithi149 ceph-mon[23948]: pgmap v7: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 5.5 MiB/s wr, 514 op/s 2024-04-22T23:14:48.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:48 smithi149 ceph-mon[23948]: Standby manager daemon smithi149.rxqkqk restarted 2024-04-22T23:14:48.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:48 smithi149 ceph-mon[23948]: Standby manager daemon smithi149.rxqkqk started 2024-04-22T23:14:48.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:48 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/2401231385' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/crt"}]: dispatch 2024-04-22T23:14:48.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:48 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/2401231385' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:14:48.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:48 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/2401231385' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/key"}]: dispatch 2024-04-22T23:14:48.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:48 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/2401231385' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:14:49.267 INFO:teuthology.orchestra.run.smithi069.stdout: 5 123630 12.13 MB/sec execute 99 sec latency 24.010 ms 2024-04-22T23:14:49.688 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:49 smithi069 ceph-mon[21859]: mgrmap e32: smithi069.lyiiqg(active, since 10s), standbys: smithi149.rxqkqk 2024-04-22T23:14:49.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:49 smithi149 ceph-mon[23948]: mgrmap e32: smithi069.lyiiqg(active, since 10s), standbys: smithi149.rxqkqk 2024-04-22T23:14:50.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 124044 12.18 MB/sec execute 100 sec latency 32.339 ms 2024-04-22T23:14:50.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:50 smithi069 ceph-mon[21859]: pgmap v8: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.3 MiB/s wr, 399 op/s 2024-04-22T23:14:50.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:50 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:50.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:50 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:50.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:50 smithi149 ceph-mon[23948]: pgmap v8: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.3 MiB/s wr, 399 op/s 2024-04-22T23:14:50.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:50 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:50.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:50 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:51.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 124440 12.14 MB/sec execute 101 sec latency 41.453 ms 2024-04-22T23:14:52.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 124963 12.15 MB/sec execute 102 sec latency 31.817 ms 2024-04-22T23:14:53.217 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:53 smithi069 ceph-mon[21859]: pgmap v9: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 7.7 MiB/s wr, 740 op/s 2024-04-22T23:14:53.217 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:53 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:53.217 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:53 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:53.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 125618 12.13 MB/sec execute 103 sec latency 30.367 ms 2024-04-22T23:14:53.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:53 smithi149 ceph-mon[23948]: pgmap v9: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 7.7 MiB/s wr, 740 op/s 2024-04-22T23:14:53.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:53 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:53.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:53 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:54.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 126055 12.11 MB/sec execute 104 sec latency 34.807 ms 2024-04-22T23:14:55.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:55 smithi069 ceph-mon[21859]: pgmap v10: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 7.0 MiB/s wr, 677 op/s 2024-04-22T23:14:55.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:55 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:14:55.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 126539 12.16 MB/sec execute 105 sec latency 41.780 ms 2024-04-22T23:14:55.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:55 smithi149 ceph-mon[23948]: pgmap v10: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 7.0 MiB/s wr, 677 op/s 2024-04-22T23:14:55.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:55 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:14:56.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 126963 12.14 MB/sec execute 106 sec latency 57.068 ms 2024-04-22T23:14:57.040 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:57 smithi069 ceph-mon[21859]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 32 KiB/s rd, 7.8 MiB/s wr, 715 op/s 2024-04-22T23:14:57.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 127508 12.24 MB/sec execute 107 sec latency 32.608 ms 2024-04-22T23:14:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:57 smithi149 ceph-mon[23948]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 32 KiB/s rd, 7.8 MiB/s wr, 715 op/s 2024-04-22T23:14:58.070 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:14:58.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 128053 12.29 MB/sec execute 108 sec latency 25.653 ms 2024-04-22T23:14:58.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:58 smithi069 ceph-mon[21859]: pgmap v12: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 8.9 MiB/s wr, 933 op/s 2024-04-22T23:14:58.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:58 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:58.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:58 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:58.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:58 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:14:58.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:58 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:14:58.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:58 smithi069 ceph-mon[21859]: Updating smithi069:/etc/ceph/ceph.conf 2024-04-22T23:14:58.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:58 smithi069 ceph-mon[21859]: Updating smithi149:/etc/ceph/ceph.conf 2024-04-22T23:14:58.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:58 smithi069 ceph-mon[21859]: Updating smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T23:14:58.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:58 smithi069 ceph-mon[21859]: Updating smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T23:14:58.818 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:58 smithi069 ceph-mon[21859]: from='client.14810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:14:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:58 smithi149 ceph-mon[23948]: pgmap v12: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 8.9 MiB/s wr, 933 op/s 2024-04-22T23:14:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:58 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:58 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:14:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:58 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:14:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:58 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:14:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:58 smithi149 ceph-mon[23948]: Updating smithi069:/etc/ceph/ceph.conf 2024-04-22T23:14:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:58 smithi149 ceph-mon[23948]: Updating smithi149:/etc/ceph/ceph.conf 2024-04-22T23:14:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:58 smithi149 ceph-mon[23948]: Updating smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T23:14:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:58 smithi149 ceph-mon[23948]: Updating smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T23:14:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:58 smithi149 ceph-mon[23948]: from='client.14810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:14:59.047 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:14:59.047 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (69s) 8s ago 10m 15.5M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:14:59.047 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (10m) 8s ago 10m 6400k - 17.2.0 e1d6a67b021e f29a69890fd2 2024-04-22T23:14:59.047 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (9m) 14s ago 9m 6400k - 17.2.0 e1d6a67b021e d3576856bdfd 2024-04-22T23:14:59.047 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (62s) 8s ago 10m 44.4M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:14:59.047 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (4m) 8s ago 4m 37.3M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:14:59.047 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (4m) 14s ago 4m 13.6M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:14:59.047 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (39s) 8s ago 11m 480M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:14:59.047 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283 running (110s) 14s ago 9m 88.0M - 19.0.0-3252-g43be0201 a0434699c1d7 43be2fc88208 2024-04-22T23:14:59.047 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (11m) 8s ago 11m 57.9M 2048M 17.2.0 e1d6a67b021e 008ad8fe38c6 2024-04-22T23:14:59.047 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (9m) 14s ago 9m 45.3M 2048M 17.2.0 e1d6a67b021e 1a87df28e650 2024-04-22T23:14:59.047 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (4m) 8s ago 4m 252M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:14:59.048 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (3m) 14s ago 3m 81.7M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:14:59.048 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (59s) 8s ago 9m 10.3M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:14:59.048 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (48s) 14s ago 9m 8690k - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:14:59.048 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (8m) 8s ago 8m 449M 1936M 17.2.0 e1d6a67b021e 777db7a3d809 2024-04-22T23:14:59.048 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (8m) 8s ago 8m 365M 1936M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:14:59.048 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (7m) 8s ago 7m 286M 1936M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:14:59.048 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (7m) 8s ago 7m 402M 1936M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:14:59.048 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (7m) 14s ago 7m 390M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:14:59.048 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (6m) 14s ago 6m 452M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:14:59.048 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (6m) 14s ago 6m 384M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:14:59.048 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (5m) 14s ago 5m 521M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:14:59.048 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (51s) 8s ago 9m 53.2M - 2.33.4 514e6a882f6e 362147bc8420 2024-04-22T23:14:59.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 128309 12.21 MB/sec execute 109 sec latency 47.719 ms 2024-04-22T23:14:59.611 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:14:59.612 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:14:59.612 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:14:59.612 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:14:59.612 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:14:59.612 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:14:59.612 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:14:59.613 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:14:59.613 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T23:14:59.613 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:14:59.613 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:14:59.613 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:14:59.613 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:14:59.613 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:14:59.614 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-22T23:14:59.614 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:14:59.614 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:14:59.614 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:15:00.080 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:15:00.080 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:15:00.080 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:15:00.080 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: Updating smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T23:15:00.080 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: Updating smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T23:15:00.080 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:00.080 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:00.080 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:00.080 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:00.080 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:00.081 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:00.081 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: from='client.14818 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:15:00.081 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:14:59 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/2042431561' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:15:00.126 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:15:00.126 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:15:00.126 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:15:00.126 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:15:00.126 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:15:00.126 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-04-22T23:15:00.127 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:15:00.127 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "2/23 daemons upgraded", 2024-04-22T23:15:00.127 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "", 2024-04-22T23:15:00.127 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:15:00.127 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:15:00.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:15:00.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:15:00.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:15:00.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: Updating smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T23:15:00.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: Updating smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T23:15:00.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:00.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:00.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:00.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:00.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:00.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:00.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: from='client.14818 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:15:00.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:14:59 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/2042431561' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:15:00.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 128571 12.14 MB/sec execute 110 sec latency 41.050 ms 2024-04-22T23:15:00.748 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:15:01.094 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:00 smithi069 ceph-mon[21859]: Reconfiguring prometheus.smithi069 (dependencies changed)... 2024-04-22T23:15:01.094 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:00 smithi069 ceph-mon[21859]: pgmap v13: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.6 MiB/s wr, 678 op/s 2024-04-22T23:15:01.094 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:00 smithi069 ceph-mon[21859]: Reconfiguring daemon prometheus.smithi069 on smithi069 2024-04-22T23:15:01.094 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:00 smithi069 ceph-mon[21859]: from='client.? 172.21.15.69:0/694324343' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:15:01.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 128905 12.10 MB/sec execute 111 sec latency 61.019 ms 2024-04-22T23:15:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:00 smithi149 ceph-mon[23948]: Reconfiguring prometheus.smithi069 (dependencies changed)... 2024-04-22T23:15:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:00 smithi149 ceph-mon[23948]: pgmap v13: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.6 MiB/s wr, 678 op/s 2024-04-22T23:15:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:00 smithi149 ceph-mon[23948]: Reconfiguring daemon prometheus.smithi069 on smithi069 2024-04-22T23:15:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:00 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/694324343' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:15:02.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:01 smithi069 ceph-mon[21859]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:15:02.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 129300 12.05 MB/sec execute 112 sec latency 36.690 ms 2024-04-22T23:15:02.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:01 smithi149 ceph-mon[23948]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:15:03.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:02 smithi069 ceph-mon[21859]: pgmap v14: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 926 op/s 2024-04-22T23:15:03.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 129604 11.99 MB/sec execute 113 sec latency 47.735 ms 2024-04-22T23:15:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:02 smithi149 ceph-mon[23948]: pgmap v14: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 926 op/s 2024-04-22T23:15:04.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 129944 11.99 MB/sec execute 114 sec latency 62.882 ms 2024-04-22T23:15:05.165 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:04 smithi069 ceph-mon[21859]: pgmap v15: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.7 MiB/s wr, 547 op/s 2024-04-22T23:15:05.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 130411 11.98 MB/sec execute 115 sec latency 38.908 ms 2024-04-22T23:15:05.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:04 smithi149 ceph-mon[23948]: pgmap v15: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.7 MiB/s wr, 547 op/s 2024-04-22T23:15:06.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 130746 12.01 MB/sec execute 116 sec latency 52.712 ms 2024-04-22T23:15:07.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:06 smithi149 ceph-mon[23948]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 679 op/s 2024-04-22T23:15:07.290 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:06 smithi069 ceph-mon[21859]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 679 op/s 2024-04-22T23:15:07.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 131112 12.04 MB/sec execute 117 sec latency 36.031 ms 2024-04-22T23:15:08.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 131397 12.00 MB/sec execute 118 sec latency 43.694 ms 2024-04-22T23:15:08.981 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:08 smithi149 ceph-mon[23948]: pgmap v17: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.8 MiB/s wr, 726 op/s 2024-04-22T23:15:08.982 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:08 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:08.982 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:08 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:08.982 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:08 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:15:08.982 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:08 smithi149 ceph-mon[23948]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:15:08.982 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:08 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi069:9095"}]: dispatch 2024-04-22T23:15:08.982 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:08 smithi149 ceph-mon[23948]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi069:9095"}]: dispatch 2024-04-22T23:15:08.982 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:08 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:08.982 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:08 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:15:09.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:08 smithi069 ceph-mon[21859]: pgmap v17: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.8 MiB/s wr, 726 op/s 2024-04-22T23:15:09.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:08 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:09.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:08 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:09.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:08 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:15:09.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:08 smithi069 ceph-mon[21859]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:15:09.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:08 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi069:9095"}]: dispatch 2024-04-22T23:15:09.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:08 smithi069 ceph-mon[21859]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi069:9095"}]: dispatch 2024-04-22T23:15:09.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:08 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:09.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:08 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:15:09.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 131833 11.99 MB/sec execute 119 sec latency 31.870 ms 2024-04-22T23:15:10.009 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:09 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:15:10.012 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:09 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:15:10.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 132239 11.95 MB/sec execute 120 sec latency 30.263 ms 2024-04-22T23:15:11.073 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:10 smithi069 ceph-mon[21859]: pgmap v18: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-04-22T23:15:11.073 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:10 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.rxqkqk", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:15:11.074 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:10 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:15:11.074 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:10 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:15:11.100 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:10 smithi149 ceph-mon[23948]: pgmap v18: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-04-22T23:15:11.101 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:10 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.rxqkqk", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:15:11.101 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:10 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:15:11.101 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:10 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:15:11.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 132525 11.90 MB/sec execute 121 sec latency 67.500 ms 2024-04-22T23:15:12.087 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:11 smithi069 ceph-mon[21859]: Upgrade: Updating mgr.smithi149.rxqkqk 2024-04-22T23:15:12.087 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:11 smithi069 ceph-mon[21859]: Deploying daemon mgr.smithi149.rxqkqk on smithi149 2024-04-22T23:15:12.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:11 smithi149 ceph-mon[23948]: Upgrade: Updating mgr.smithi149.rxqkqk 2024-04-22T23:15:12.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:11 smithi149 ceph-mon[23948]: Deploying daemon mgr.smithi149.rxqkqk on smithi149 2024-04-22T23:15:12.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 132862 11.85 MB/sec execute 122 sec latency 53.500 ms 2024-04-22T23:15:12.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:12 smithi069 ceph-mon[21859]: pgmap v19: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.4 MiB/s wr, 692 op/s 2024-04-22T23:15:13.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:12 smithi149 ceph-mon[23948]: pgmap v19: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.4 MiB/s wr, 692 op/s 2024-04-22T23:15:13.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 133163 11.81 MB/sec execute 123 sec latency 63.874 ms 2024-04-22T23:15:14.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 133418 11.77 MB/sec execute 124 sec latency 86.548 ms 2024-04-22T23:15:14.932 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:14 smithi069 ceph-mon[21859]: pgmap v20: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 444 op/s 2024-04-22T23:15:14.932 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:14 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:14.933 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:14 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:14.933 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:14 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:15:15.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:14 smithi149 ceph-mon[23948]: pgmap v20: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 444 op/s 2024-04-22T23:15:15.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:14 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:15.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:14 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:15.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:14 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:15:15.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 133808 11.76 MB/sec execute 125 sec latency 69.185 ms 2024-04-22T23:15:16.268 INFO:teuthology.orchestra.run.smithi069.stdout: 5 134125 11.75 MB/sec execute 126 sec latency 72.104 ms 2024-04-22T23:15:16.478 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:16 smithi069 ceph-mon[21859]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.8 MiB/s wr, 474 op/s 2024-04-22T23:15:16.479 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:16 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:16.479 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:16 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:16.608 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:16 smithi149 ceph-mon[23948]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.8 MiB/s wr, 474 op/s 2024-04-22T23:15:16.608 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:16 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:16.609 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:16 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:17.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 134402 11.74 MB/sec execute 127 sec latency 58.605 ms 2024-04-22T23:15:18.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 134722 11.77 MB/sec execute 128 sec latency 37.343 ms 2024-04-22T23:15:18.352 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:18 smithi069 ceph-mon[21859]: pgmap v22: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-04-22T23:15:18.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:18 smithi149 ceph-mon[23948]: pgmap v22: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-04-22T23:15:19.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 135077 11.75 MB/sec execute 129 sec latency 33.949 ms 2024-04-22T23:15:20.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 135389 11.71 MB/sec execute 130 sec latency 41.569 ms 2024-04-22T23:15:20.634 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:20 smithi069 ceph-mon[21859]: pgmap v23: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-04-22T23:15:20.635 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:20 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:20.635 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:20 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:20.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:20 smithi149 ceph-mon[23948]: pgmap v23: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-04-22T23:15:20.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:20 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:20.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:20 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:21.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 135650 11.65 MB/sec execute 131 sec latency 50.761 ms 2024-04-22T23:15:22.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 135883 11.62 MB/sec execute 132 sec latency 91.715 ms 2024-04-22T23:15:22.980 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:22 smithi069 ceph-mon[21859]: pgmap v24: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.6 MiB/s wr, 611 op/s 2024-04-22T23:15:22.980 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:22 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:22.980 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:22 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:23.030 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:22 smithi149 ceph-mon[23948]: pgmap v24: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.6 MiB/s wr, 611 op/s 2024-04-22T23:15:23.030 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:22 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:23.031 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:22 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:23.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 136099 11.55 MB/sec execute 133 sec latency 79.970 ms 2024-04-22T23:15:24.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:24 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:24.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:24 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:24.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:24 smithi069 ceph-mon[21859]: pgmap v25: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 383 op/s 2024-04-22T23:15:24.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:24 smithi069 ceph-mon[21859]: Standby manager daemon smithi149.rxqkqk restarted 2024-04-22T23:15:24.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:24 smithi069 ceph-mon[21859]: Standby manager daemon smithi149.rxqkqk started 2024-04-22T23:15:24.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:24 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/1622587835' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/crt"}]: dispatch 2024-04-22T23:15:24.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:24 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/1622587835' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:15:24.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:24 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/1622587835' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/key"}]: dispatch 2024-04-22T23:15:24.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:24 smithi069 ceph-mon[21859]: from='mgr.? 172.21.15.149:0/1622587835' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:15:24.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 136386 11.51 MB/sec execute 134 sec latency 60.472 ms 2024-04-22T23:15:24.299 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:24 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:24.300 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:24 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:24.300 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:24 smithi149 ceph-mon[23948]: pgmap v25: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 383 op/s 2024-04-22T23:15:24.300 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:24 smithi149 ceph-mon[23948]: Standby manager daemon smithi149.rxqkqk restarted 2024-04-22T23:15:24.300 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:24 smithi149 ceph-mon[23948]: Standby manager daemon smithi149.rxqkqk started 2024-04-22T23:15:24.300 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:24 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/1622587835' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/crt"}]: dispatch 2024-04-22T23:15:24.300 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:24 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/1622587835' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:15:24.300 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:24 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/1622587835' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/key"}]: dispatch 2024-04-22T23:15:24.300 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:24 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/1622587835' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:15:25.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 136620 11.45 MB/sec execute 135 sec latency 81.256 ms 2024-04-22T23:15:25.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:25 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:15:25.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:25 smithi069 ceph-mon[21859]: mgrmap e33: smithi069.lyiiqg(active, since 46s), standbys: smithi149.rxqkqk 2024-04-22T23:15:25.388 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:25 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:15:25.388 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:25 smithi149 ceph-mon[23948]: mgrmap e33: smithi069.lyiiqg(active, since 46s), standbys: smithi149.rxqkqk 2024-04-22T23:15:26.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 136981 11.45 MB/sec execute 136 sec latency 43.046 ms 2024-04-22T23:15:26.811 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.3 MiB/s wr, 409 op/s 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: Upgrade: Setting container_image for all mgr 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi069.lyiiqg"}]: dispatch 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi069.lyiiqg"}]': finished 2024-04-22T23:15:26.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi149.rxqkqk"}]: dispatch 2024-04-22T23:15:26.813 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi149.rxqkqk"}]': finished 2024-04-22T23:15:26.813 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:26 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T23:15:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.3 MiB/s wr, 409 op/s 2024-04-22T23:15:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:15:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:15:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:15:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:15:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: Upgrade: Setting container_image for all mgr 2024-04-22T23:15:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi069.lyiiqg"}]: dispatch 2024-04-22T23:15:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi069.lyiiqg"}]': finished 2024-04-22T23:15:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi149.rxqkqk"}]: dispatch 2024-04-22T23:15:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi149.rxqkqk"}]': finished 2024-04-22T23:15:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:26 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T23:15:27.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 137570 11.48 MB/sec execute 137 sec latency 25.399 ms 2024-04-22T23:15:28.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 137962 11.50 MB/sec execute 138 sec latency 36.980 ms 2024-04-22T23:15:28.835 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:28 smithi069 ceph-mon[21859]: pgmap v27: 97 pgs: 97 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.8 MiB/s wr, 517 op/s 2024-04-22T23:15:29.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:28 smithi149 ceph-mon[23948]: pgmap v27: 97 pgs: 97 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.8 MiB/s wr, 517 op/s 2024-04-22T23:15:29.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 138369 11.54 MB/sec execute 139 sec latency 37.898 ms 2024-04-22T23:15:30.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:30 smithi069 ceph-mon[21859]: Upgrade: Updating mon.smithi069 2024-04-22T23:15:30.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:30 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:30.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:30 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:15:30.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:30 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T23:15:30.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:30 smithi069 ceph-mon[21859]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:15:30.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:30 smithi069 ceph-mon[21859]: Deploying daemon mon.smithi069 on smithi069 2024-04-22T23:15:30.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 138721 11.53 MB/sec execute 140 sec latency 42.663 ms 2024-04-22T23:15:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:30 smithi149 ceph-mon[23948]: Upgrade: Updating mon.smithi069 2024-04-22T23:15:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:30 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:30 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:15:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:30 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T23:15:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:30 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:15:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:30 smithi149 ceph-mon[23948]: Deploying daemon mon.smithi069 on smithi069 2024-04-22T23:15:31.129 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:31 smithi069 ceph-mon[21859]: pgmap v28: 97 pgs: 97 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 352 op/s 2024-04-22T23:15:31.248 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:15:31.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 139000 11.48 MB/sec execute 141 sec latency 40.996 ms 2024-04-22T23:15:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:31 smithi149 ceph-mon[23948]: pgmap v28: 97 pgs: 97 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 352 op/s 2024-04-22T23:15:31.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:31 smithi069 systemd[1]: Stopping Ceph mon.smithi069 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:15:32.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:32 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069[21855]: 2024-04-22T23:15:32.128+0000 7f37ce824700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi069 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:15:32.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:32 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069[21855]: 2024-04-22T23:15:32.128+0000 7f37ce824700 -1 mon.smithi069@0(leader) e2 *** Got Signal Terminated *** 2024-04-22T23:15:32.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 139241 11.42 MB/sec execute 142 sec latency 67.114 ms 2024-04-22T23:15:32.293 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:15:32.293 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (102s) 9s ago 10m 23.7M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:15:32.293 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (10m) 9s ago 10m 6400k - 17.2.0 e1d6a67b021e f29a69890fd2 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (10m) 12s ago 10m 6400k - 17.2.0 e1d6a67b021e d3576856bdfd 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (95s) 9s ago 10m 44.3M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (4m) 9s ago 4m 37.5M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (4m) 11s ago 4m 13.8M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (73s) 9s ago 12m 495M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (18s) 12s ago 10m 99.8M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (12m) 9s ago 12m 57.2M 2048M 17.2.0 e1d6a67b021e 008ad8fe38c6 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (10m) 12s ago 10m 43.7M 2048M 17.2.0 e1d6a67b021e 1a87df28e650 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (4m) 9s ago 4m 246M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (4m) 11s ago 4m 83.8M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (92s) 9s ago 10m 11.0M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (81s) 12s ago 9m 8556k - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:15:32.294 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (9m) 9s ago 9m 478M 1936M 17.2.0 e1d6a67b021e 777db7a3d809 2024-04-22T23:15:32.295 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (8m) 9s ago 8m 411M 1936M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:15:32.295 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (8m) 9s ago 8m 298M 1936M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:15:32.295 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (7m) 9s ago 7m 433M 1936M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:15:32.295 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (7m) 12s ago 7m 459M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:15:32.295 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (7m) 12s ago 7m 512M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:15:32.295 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (6m) 11s ago 6m 425M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:15:32.295 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (6m) 11s ago 6m 327M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:15:32.295 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (25s) 9s ago 9m 50.7M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:15:32.400 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:32 smithi069 podman[62901]: 2024-04-22 23:15:32.163200627 +0000 UTC m=+0.195486742 container died 008ad8fe38c6996b05863941b6022e3e37280cf81fb93b1d02ce3e90d6e4800d (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, name=centos-stream, ceph=True, io.openshift.expose-services=, release=754, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD) 2024-04-22T23:15:32.937 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:15:32.937 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:15:32.938 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:15:32.938 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:15:32.938 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:15:32.938 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:15:32.938 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:15:32.938 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:15:32.938 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T23:15:32.938 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:15:32.938 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:15:32.938 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:15:32.938 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:15:32.939 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:15:32.939 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-22T23:15:32.939 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:15:32.939 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:15:32.939 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:15:33.169 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:32 smithi069 podman[62901]: 2024-04-22 23:15:32.934007782 +0000 UTC m=+0.966293908 container cleanup 008ad8fe38c6996b05863941b6022e3e37280cf81fb93b1d02ce3e90d6e4800d (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.buildah.version=1.19.8, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., distribution-scope=public, GIT_BRANCH=HEAD, version=8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, release=754) 2024-04-22T23:15:33.169 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:32 smithi069 bash[62901]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069 2024-04-22T23:15:33.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 139571 11.40 MB/sec execute 143 sec latency 57.938 ms 2024-04-22T23:15:33.549 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:15:33.549 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:15:33.549 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:15:33.549 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:15:33.549 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:15:33.550 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-04-22T23:15:33.550 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:15:33.550 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "2/23 daemons upgraded", 2024-04-22T23:15:33.550 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading mon daemons", 2024-04-22T23:15:33.550 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:15:33.550 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:15:33.672 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:33 smithi069 podman[62913]: 2024-04-22 23:15:33.61365033 +0000 UTC m=+1.442860684 container remove 008ad8fe38c6996b05863941b6022e3e37280cf81fb93b1d02ce3e90d6e4800d (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, distribution-scope=public, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_BRANCH=HEAD, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, release=754) 2024-04-22T23:15:33.925 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:33 smithi069 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi069.service: Deactivated successfully. 2024-04-22T23:15:33.925 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:33 smithi069 systemd[1]: Stopped Ceph mon.smithi069 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:15:33.925 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:33 smithi069 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi069.service: Consumed 15.983s CPU time. 2024-04-22T23:15:34.184 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:15:34.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 139915 11.37 MB/sec execute 144 sec latency 53.550 ms 2024-04-22T23:15:34.428 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 systemd[1]: Starting Ceph mon.smithi069 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:15:34.691 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 podman[63116]: 2024-04-22 23:15:34.492114737 +0000 UTC m=+0.047980408 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:15:34.691 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 podman[63116]: 2024-04-22 23:15:34.6064757 +0000 UTC m=+0.162341366 container create e4e20d532cda9bf38ecd44f908361cd0c377b3d5aeff38f7e11478e11a0b1c7b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True) 2024-04-22T23:15:34.945 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 podman[63116]: 2024-04-22 23:15:34.884576347 +0000 UTC m=+0.440442021 container init e4e20d532cda9bf38ecd44f908361cd0c377b3d5aeff38f7e11478e11a0b1c7b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-22T23:15:34.945 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 podman[63116]: 2024-04-22 23:15:34.89335167 +0000 UTC m=+0.449217334 container start e4e20d532cda9bf38ecd44f908361cd0c377b3d5aeff38f7e11478e11a0b1c7b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-22T23:15:34.945 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T23:15:34.945 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev), process ceph-mon, pid 2 2024-04-22T23:15:34.945 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: pidfile_write: ignore empty --pid-file 2024-04-22T23:15:34.945 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: load: jerasure load: lrc 2024-04-22T23:15:34.945 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: RocksDB version: 7.9.2 2024-04-22T23:15:34.945 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Git sha 0 2024-04-22T23:15:34.945 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Compile date 2024-04-22 21:33:31 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: DB SUMMARY 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: DB Session ID: KP6SJ4W6R4MQ2Q2E2G2V 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: CURRENT file: CURRENT 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: MANIFEST file: MANIFEST-000015 size: 979 Bytes 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi069/store.db dir, Total Num: 1, files: 000030.sst 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi069/store.db: 000028.log size: 1605234 ; 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.error_if_exists: 0 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.create_if_missing: 0 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.paranoid_checks: 1 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T23:15:34.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.env: 0x55ddf33fbcc0 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.fs: PosixFileSystem 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.info_log: 0x55ddf5629aa0 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.statistics: (nil) 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.use_fsync: 0 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_log_file_size: 0 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.allow_fallocate: 1 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T23:15:34.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.use_direct_reads: 0 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.db_log_dir: 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.wal_dir: 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.write_buffer_manager: 0x55ddf55bd0e0 2024-04-22T23:15:34.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.unordered_write: 0 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.row_cache: None 2024-04-22T23:15:34.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.wal_filter: None 2024-04-22T23:15:34.950 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T23:15:34.950 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T23:15:34.950 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.two_write_queues: 0 2024-04-22T23:15:34.950 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T23:15:34.950 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.wal_compression: 0 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.atomic_flush: 0 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.log_readahead_size: 0 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_background_jobs: 2 2024-04-22T23:15:34.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_background_compactions: -1 2024-04-22T23:15:34.952 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_subcompactions: 1 2024-04-22T23:15:34.952 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T23:15:34.952 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T23:15:34.952 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T23:15:34.952 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T23:15:34.953 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T23:15:34.953 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T23:15:34.953 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T23:15:34.953 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T23:15:34.953 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_open_files: -1 2024-04-22T23:15:34.953 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_background_flushes: -1 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Compression algorithms supported: 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: kZSTD supported: 0 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: kXpressCompression supported: 0 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: kZlibCompression supported: 1 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: kSnappyCompression supported: 1 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: kLZ4Compression supported: 1 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: kBZip2Compression supported: 0 2024-04-22T23:15:34.954 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi069/store.db/MANIFEST-000015 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.merge_operator: 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_filter: None 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ddf5629c80) 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: cache_index_and_filter_blocks: 1 2024-04-22T23:15:34.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: pin_top_level_index_and_filter: 1 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_type: 0 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: data_block_index_type: 0 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_shortening: 1 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: checksum: 4 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: no_block_cache: 0 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache: 0x55ddf55ef1f0 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_name: BinnedLRUCache 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_options: 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: capacity : 536870912 2024-04-22T23:15:34.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: num_shard_bits : 4 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: strict_capacity_limit : 0 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: high_pri_pool_ratio: 0.000 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_compressed: (nil) 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: persistent_cache: (nil) 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_size: 4096 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_size_deviation: 10 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_restart_interval: 16 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_block_restart_interval: 1 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: metadata_block_size: 4096 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: partition_filters: 0 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: use_delta_encoding: 1 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: filter_policy: bloomfilter 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: whole_key_filtering: 1 2024-04-22T23:15:34.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: verify_compression: 0 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: read_amp_bytes_per_bit: 0 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: format_version: 5 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: enable_index_compression: 1 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_align: 0 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: max_auto_readahead_size: 262144 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: prepopulate_block_cache: 0 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: initial_auto_readahead_size: 8192 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: num_file_reads_for_auto_readahead: 2 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compression: NoCompression 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T23:15:34.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.num_levels: 7 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T23:15:34.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T23:15:34.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.arena_block_size: 1048576 2024-04-22T23:15:34.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T23:15:34.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T23:15:34.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T23:15:34.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T23:15:34.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T23:15:34.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T23:15:34.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T23:15:34.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T23:15:34.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T23:15:34.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T23:15:34.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T23:15:34.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T23:15:34.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T23:15:34.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-22T23:15:34.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.inplace_update_support: 0 2024-04-22T23:15:34.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T23:15:34.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T23:15:34.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T23:15:34.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T23:15:34.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.bloom_locality: 0 2024-04-22T23:15:34.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.max_successive_merges: 0 2024-04-22T23:15:34.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T23:15:34.963 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.ttl: 2592000 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.enable_blob_files: false 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.min_blob_size: 0 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T23:15:34.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-22T23:15:34.965 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-22T23:15:34.965 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.blob_file_starting_level: 0 2024-04-22T23:15:35.193 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-22T23:15:35.193 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-04-22T23:15:35.193 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T23:15:35.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi069/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11851, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T23:15:35.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-04-22T23:15:35.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 4f7ca0af-9b79-4405-8e30-f828ef24bef8 2024-04-22T23:15:35.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713827734945142, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-04-22T23:15:35.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-04-22T23:15:35.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713827734952803, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 1428022, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11852, "largest_seqno": 12706, "table_properties": {"data_size": 1423455, "index_size": 2567, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 965, "raw_key_size": 8527, "raw_average_key_size": 23, "raw_value_size": 1415289, "raw_average_value_size": 3920, "num_data_blocks": 117, "num_entries": 361, "num_filter_entries": 361, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1713827734, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "4f7ca0af-9b79-4405-8e30-f828ef24bef8", "db_session_id": "KP6SJ4W6R4MQ2Q2E2G2V", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-04-22T23:15:35.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713827734953023, "job": 1, "event": "recovery_finished"} 2024-04-22T23:15:35.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-04-22T23:15:35.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T23:15:35.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi069/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T23:15:35.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ddf5704000 2024-04-22T23:15:35.194 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: DB pointer 0x55ddf56ee000 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** DB Stats ** 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T23:15:35.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: L0 1/0 1.36 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 331.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: L6 1/0 7.56 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Sum 2/0 8.92 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 331.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 331.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 331.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T23:15:35.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative compaction: 0.00 GB write, 97.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval compaction: 0.00 GB write, 97.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Block cache BinnedLRUCache@0x55ddf55ef1f0#2 capacity: 512.00 MB usage: 3.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: starting mon.smithi069 rank 0 at public addrs [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] at bind addrs [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi069 fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:15:35.197 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: mon.smithi069@-1(???) e2 preinit fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:15:35.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: mon.smithi069@-1(???).mds e7 new map 2024-04-22T23:15:35.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: mon.smithi069@-1(???).mds e7 print_map 2024-04-22T23:15:35.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: e7 2024-04-22T23:15:35.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T23:15:35.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T23:15:35.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: legacy client fscid: 1 2024-04-22T23:15:35.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:15:35.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Filesystem 'foofs' (1) 2024-04-22T23:15:35.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: fs_name foofs 2024-04-22T23:15:35.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: epoch 5 2024-04-22T23:15:35.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: created 2024-04-22T23:10:32.113295+0000 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: modified 2024-04-22T23:10:38.133401+0000 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: tableserver 0 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: root 0 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: session_timeout 60 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: session_autoclose 300 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: max_file_size 1099511627776 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: max_xattr_size 65536 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: required_client_features {} 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: last_failure 0 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: last_failure_osd_epoch 0 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T23:15:35.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: max_mds 1 2024-04-22T23:15:35.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: in 0 2024-04-22T23:15:35.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: up {0=14498} 2024-04-22T23:15:35.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: failed 2024-04-22T23:15:35.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: damaged 2024-04-22T23:15:35.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: stopped 2024-04-22T23:15:35.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: data_pools [3] 2024-04-22T23:15:35.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: metadata_pool 2 2024-04-22T23:15:35.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: inline_data disabled 2024-04-22T23:15:35.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: balancer 2024-04-22T23:15:35.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: bal_rank_mask -1 2024-04-22T23:15:35.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: standby_count_wanted 1 2024-04-22T23:15:35.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: qdb_cluster leader: 0 members: 2024-04-22T23:15:35.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: [mds.foofs.smithi069.jiyres{0:14498} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.69:6834/1879302209,v1:172.21.15.69:6835/1879302209] compat {c=[1],r=[1],i=[7ff]}] 2024-04-22T23:15:35.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:15:35.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:15:35.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Standby daemons: 2024-04-22T23:15:35.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-22T23:15:35.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: [mds.foofs.smithi149.ickhha{-1:24251} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.149:6832/1034958666,v1:172.21.15.149:6833/1034958666] compat {c=[1],r=[1],i=[7ff]}] 2024-04-22T23:15:35.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: mon.smithi069@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-22T23:15:35.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: mon.smithi069@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T23:15:35.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: mon.smithi069@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T23:15:35.202 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: mon.smithi069@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T23:15:35.202 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:34 smithi069 ceph-mon[63133]: mon.smithi069@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-04-22T23:15:35.202 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 bash[63116]: e4e20d532cda9bf38ecd44f908361cd0c377b3d5aeff38f7e11478e11a0b1c7b 2024-04-22T23:15:35.202 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 systemd[1]: Started Ceph mon.smithi069 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:15:35.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 140358 11.35 MB/sec execute 145 sec latency 67.452 ms 2024-04-22T23:15:35.392 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:15:35.392 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:15:35.393 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: from='client.24553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:15:35.393 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/877744652' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:15:35.393 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: pgmap v30: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 398 op/s 2024-04-22T23:15:35.393 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:15:35.393 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: from='client.? 172.21.15.69:0/113015913' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:15:35.393 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: mon.smithi069 calling monitor election 2024-04-22T23:15:35.393 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: mon.smithi069 is new leader, mons smithi069,smithi149 in quorum (ranks 0,1) 2024-04-22T23:15:35.393 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: monmap epoch 2 2024-04-22T23:15:35.393 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:15:35.393 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: last_changed 2024-04-22T23:05:26.301919+0000 2024-04-22T23:15:35.393 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: created 2024-04-22T23:03:14.830052+0000 2024-04-22T23:15:35.393 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: min_mon_release 17 (quincy) 2024-04-22T23:15:35.394 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: election_strategy: 1 2024-04-22T23:15:35.394 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: 0: [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon.smithi069 2024-04-22T23:15:35.394 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: 1: [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] mon.smithi149 2024-04-22T23:15:35.394 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 1 up:standby 2024-04-22T23:15:35.394 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T23:15:35.394 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: mgrmap e33: smithi069.lyiiqg(active, since 56s), standbys: smithi149.rxqkqk 2024-04-22T23:15:35.394 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: overall HEALTH_OK 2024-04-22T23:15:35.394 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: mgrmap e34: smithi069.lyiiqg(active, since 56s), standbys: smithi149.rxqkqk 2024-04-22T23:15:35.394 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:35 smithi149 ceph-mon[23948]: from='mgr.14758 ' entity='' 2024-04-22T23:15:35.444 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:15:35.444 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: from='mgr.14758 172.21.15.69:0/2343891866' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:15:35.444 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: from='client.24553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:15:35.444 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/877744652' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:15:35.444 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: pgmap v30: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 398 op/s 2024-04-22T23:15:35.444 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:15:35.444 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/113015913' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:15:35.444 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: mon.smithi069 calling monitor election 2024-04-22T23:15:35.444 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: mon.smithi069 is new leader, mons smithi069,smithi149 in quorum (ranks 0,1) 2024-04-22T23:15:35.444 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: monmap epoch 2 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: last_changed 2024-04-22T23:05:26.301919+0000 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: created 2024-04-22T23:03:14.830052+0000 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: min_mon_release 17 (quincy) 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: election_strategy: 1 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: 0: [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon.smithi069 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: 1: [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] mon.smithi149 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 1 up:standby 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: mgrmap e33: smithi069.lyiiqg(active, since 56s), standbys: smithi149.rxqkqk 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: overall HEALTH_OK 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: mgrmap e34: smithi069.lyiiqg(active, since 56s), standbys: smithi149.rxqkqk 2024-04-22T23:15:35.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:35 smithi069 ceph-mon[63133]: from='mgr.14758 ' entity='' 2024-04-22T23:15:36.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 141069 11.41 MB/sec execute 146 sec latency 37.693 ms 2024-04-22T23:15:37.201 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:37 smithi069 ceph-mon[63133]: mgrmap e35: smithi069.lyiiqg(active, since 57s), standbys: smithi149.rxqkqk 2024-04-22T23:15:37.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 141673 11.49 MB/sec execute 147 sec latency 23.989 ms 2024-04-22T23:15:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:37 smithi149 ceph-mon[23948]: mgrmap e35: smithi069.lyiiqg(active, since 57s), standbys: smithi149.rxqkqk 2024-04-22T23:15:38.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 142374 11.56 MB/sec execute 148 sec latency 27.444 ms 2024-04-22T23:15:39.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 142942 11.58 MB/sec execute 149 sec latency 17.397 ms 2024-04-22T23:15:40.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 143500 11.56 MB/sec execute 150 sec latency 50.866 ms 2024-04-22T23:15:41.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 144002 11.58 MB/sec execute 151 sec latency 38.628 ms 2024-04-22T23:15:42.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 144492 11.57 MB/sec execute 152 sec latency 68.231 ms 2024-04-22T23:15:43.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 144927 11.59 MB/sec execute 153 sec latency 28.154 ms 2024-04-22T23:15:44.036 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:43 smithi069 ceph-mon[63133]: Standby manager daemon smithi149.rxqkqk restarted 2024-04-22T23:15:44.036 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:43 smithi069 ceph-mon[63133]: Standby manager daemon smithi149.rxqkqk started 2024-04-22T23:15:44.036 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:43 smithi069 ceph-mon[63133]: from='mgr.? 172.21.15.149:0/4245684903' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/crt"}]: dispatch 2024-04-22T23:15:44.036 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:43 smithi069 ceph-mon[63133]: from='mgr.? 172.21.15.149:0/4245684903' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:15:44.036 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:43 smithi069 ceph-mon[63133]: from='mgr.? 172.21.15.149:0/4245684903' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/key"}]: dispatch 2024-04-22T23:15:44.036 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:43 smithi069 ceph-mon[63133]: from='mgr.? 172.21.15.149:0/4245684903' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:15:44.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:43 smithi149 ceph-mon[23948]: Standby manager daemon smithi149.rxqkqk restarted 2024-04-22T23:15:44.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:43 smithi149 ceph-mon[23948]: Standby manager daemon smithi149.rxqkqk started 2024-04-22T23:15:44.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:43 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/4245684903' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/crt"}]: dispatch 2024-04-22T23:15:44.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:43 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/4245684903' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T23:15:44.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:43 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/4245684903' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi149.rxqkqk/key"}]: dispatch 2024-04-22T23:15:44.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:43 smithi149 ceph-mon[23948]: from='mgr.? 172.21.15.149:0/4245684903' entity='mgr.smithi149.rxqkqk' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T23:15:44.269 INFO:teuthology.orchestra.run.smithi069.stdout: 5 145537 11.67 MB/sec execute 154 sec latency 34.133 ms 2024-04-22T23:15:45.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:44 smithi069 ceph-mon[63133]: mgrmap e36: smithi069.lyiiqg(active, since 65s), standbys: smithi149.rxqkqk 2024-04-22T23:15:45.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:44 smithi149 ceph-mon[23948]: mgrmap e36: smithi069.lyiiqg(active, since 65s), standbys: smithi149.rxqkqk 2024-04-22T23:15:45.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 146096 11.68 MB/sec execute 155 sec latency 21.991 ms 2024-04-22T23:15:46.024 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: Active manager daemon smithi069.lyiiqg restarted 2024-04-22T23:15:46.024 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: Activating manager daemon smithi069.lyiiqg 2024-04-22T23:15:46.024 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T23:15:46.025 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: mgrmap e37: smithi069.lyiiqg(active, starting, since 0.0103004s), standbys: smithi149.rxqkqk 2024-04-22T23:15:46.025 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:15:46.025 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:15:46.025 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:15:46.025 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:15:46.025 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi069.lyiiqg", "id": "smithi069.lyiiqg"}]: dispatch 2024-04-22T23:15:46.025 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi149.rxqkqk", "id": "smithi149.rxqkqk"}]: dispatch 2024-04-22T23:15:46.025 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:15:46.025 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: Manager daemon smithi069.lyiiqg is now available 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:15:46.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:15:46.135 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: Active manager daemon smithi069.lyiiqg restarted 2024-04-22T23:15:46.135 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: Activating manager daemon smithi069.lyiiqg 2024-04-22T23:15:46.135 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T23:15:46.135 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: mgrmap e37: smithi069.lyiiqg(active, starting, since 0.0103004s), standbys: smithi149.rxqkqk 2024-04-22T23:15:46.135 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:15:46.135 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:15:46.135 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:15:46.136 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:15:46.136 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi069.lyiiqg", "id": "smithi069.lyiiqg"}]: dispatch 2024-04-22T23:15:46.136 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr metadata", "who": "smithi149.rxqkqk", "id": "smithi149.rxqkqk"}]: dispatch 2024-04-22T23:15:46.136 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:15:46.136 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:15:46.136 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:15:46.136 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:15:46.136 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:15:46.137 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:15:46.137 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:15:46.137 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:15:46.137 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T23:15:46.137 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T23:15:46.137 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T23:15:46.137 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: Manager daemon smithi069.lyiiqg is now available 2024-04-22T23:15:46.137 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:15:46.137 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:15:46.137 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:45 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/mirror_snapshot_schedule"}]: dispatch 2024-04-22T23:15:46.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 146355 11.64 MB/sec execute 156 sec latency 59.349 ms 2024-04-22T23:15:47.027 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:15:47.028 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:46 smithi069 ceph-mon[63133]: mgrmap e38: smithi069.lyiiqg(active, since 1.01565s), standbys: smithi149.rxqkqk 2024-04-22T23:15:47.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:46 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.lyiiqg/trash_purge_schedule"}]: dispatch 2024-04-22T23:15:47.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:46 smithi149 ceph-mon[23948]: mgrmap e38: smithi069.lyiiqg(active, since 1.01565s), standbys: smithi149.rxqkqk 2024-04-22T23:15:47.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 146761 11.61 MB/sec execute 157 sec latency 29.549 ms 2024-04-22T23:15:48.057 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:47 smithi069 ceph-mon[63133]: [22/Apr/2024:23:15:47] ENGINE Bus STARTING 2024-04-22T23:15:48.057 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:47 smithi069 ceph-mon[63133]: [22/Apr/2024:23:15:47] ENGINE Serving on https://172.21.15.69:7150 2024-04-22T23:15:48.184 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:47 smithi149 ceph-mon[23948]: [22/Apr/2024:23:15:47] ENGINE Bus STARTING 2024-04-22T23:15:48.184 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:47 smithi149 ceph-mon[23948]: [22/Apr/2024:23:15:47] ENGINE Serving on https://172.21.15.69:7150 2024-04-22T23:15:48.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 147289 11.60 MB/sec execute 158 sec latency 38.861 ms 2024-04-22T23:15:48.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:48 smithi069 ceph-mon[63133]: [22/Apr/2024:23:15:47] ENGINE Serving on http://172.21.15.69:8765 2024-04-22T23:15:48.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:48 smithi069 ceph-mon[63133]: [22/Apr/2024:23:15:47] ENGINE Bus STARTED 2024-04-22T23:15:48.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:48 smithi069 ceph-mon[63133]: pgmap v4: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-22T23:15:48.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:48 smithi069 ceph-mon[63133]: mgrmap e39: smithi069.lyiiqg(active, since 2s), standbys: smithi149.rxqkqk 2024-04-22T23:15:49.175 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:48 smithi149 ceph-mon[23948]: [22/Apr/2024:23:15:47] ENGINE Serving on http://172.21.15.69:8765 2024-04-22T23:15:49.175 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:48 smithi149 ceph-mon[23948]: [22/Apr/2024:23:15:47] ENGINE Bus STARTED 2024-04-22T23:15:49.175 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:48 smithi149 ceph-mon[23948]: pgmap v4: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-22T23:15:49.175 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:48 smithi149 ceph-mon[23948]: mgrmap e39: smithi069.lyiiqg(active, since 2s), standbys: smithi149.rxqkqk 2024-04-22T23:15:49.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 148032 11.66 MB/sec execute 159 sec latency 27.153 ms 2024-04-22T23:15:50.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 148529 11.69 MB/sec execute 160 sec latency 26.787 ms 2024-04-22T23:15:51.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:50 smithi069 ceph-mon[63133]: pgmap v5: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-22T23:15:51.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:50 smithi069 ceph-mon[63133]: mgrmap e40: smithi069.lyiiqg(active, since 4s), standbys: smithi149.rxqkqk 2024-04-22T23:15:51.124 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:50 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:51.124 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:50 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:50 smithi149 ceph-mon[23948]: pgmap v5: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-22T23:15:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:50 smithi149 ceph-mon[23948]: mgrmap e40: smithi069.lyiiqg(active, since 4s), standbys: smithi149.rxqkqk 2024-04-22T23:15:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:50 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:50 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:51.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 148882 11.73 MB/sec execute 161 sec latency 42.474 ms 2024-04-22T23:15:52.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 149162 11.69 MB/sec execute 162 sec latency 64.640 ms 2024-04-22T23:15:53.155 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:53 smithi069 ceph-mon[63133]: pgmap v6: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 41 KiB/s rd, 6.0 MiB/s wr, 660 op/s 2024-04-22T23:15:53.155 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:53 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:53.156 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:53 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:53.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 149422 11.67 MB/sec execute 163 sec latency 50.644 ms 2024-04-22T23:15:53.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:53 smithi149 ceph-mon[23948]: pgmap v6: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 41 KiB/s rd, 6.0 MiB/s wr, 660 op/s 2024-04-22T23:15:53.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:53 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:53.330 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:53 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:54.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 149766 11.63 MB/sec execute 164 sec latency 38.558 ms 2024-04-22T23:15:55.177 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:55 smithi069 ceph-mon[63133]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 36 KiB/s rd, 5.1 MiB/s wr, 507 op/s 2024-04-22T23:15:55.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 150253 11.62 MB/sec execute 165 sec latency 39.344 ms 2024-04-22T23:15:55.350 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:55 smithi149 ceph-mon[23948]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 36 KiB/s rd, 5.1 MiB/s wr, 507 op/s 2024-04-22T23:15:56.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:56.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:56.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 150663 11.60 MB/sec execute 166 sec latency 35.060 ms 2024-04-22T23:15:56.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:56 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:56.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:56 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:57.212 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:57 smithi069 ceph-mon[63133]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 3.9 MiB/s wr, 394 op/s 2024-04-22T23:15:57.213 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:57 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:57.213 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:57 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:57.213 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:57 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:15:57.213 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:57 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:57.213 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:57 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:57.213 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:57 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:15:57.213 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:57 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:15:57.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 151144 11.61 MB/sec execute 167 sec latency 37.103 ms 2024-04-22T23:15:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:57 smithi149 ceph-mon[23948]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 3.9 MiB/s wr, 394 op/s 2024-04-22T23:15:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:57 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:57 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:57 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:15:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:57 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:57 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:57 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:15:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:57 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:15:58.236 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:58 smithi069 ceph-mon[63133]: Updating smithi069:/etc/ceph/ceph.conf 2024-04-22T23:15:58.237 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:58 smithi069 ceph-mon[63133]: Updating smithi149:/etc/ceph/ceph.conf 2024-04-22T23:15:58.237 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:58 smithi069 ceph-mon[63133]: Updating smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T23:15:58.237 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:58 smithi069 ceph-mon[63133]: Updating smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T23:15:58.237 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:58 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:58.237 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:58 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:58.237 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:58 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:58.237 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:58 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:58.237 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:58 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:58.237 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:58 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:58.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 151606 11.60 MB/sec execute 168 sec latency 42.792 ms 2024-04-22T23:15:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:58 smithi149 ceph-mon[23948]: Updating smithi069:/etc/ceph/ceph.conf 2024-04-22T23:15:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:58 smithi149 ceph-mon[23948]: Updating smithi149:/etc/ceph/ceph.conf 2024-04-22T23:15:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:58 smithi149 ceph-mon[23948]: Updating smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T23:15:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:58 smithi149 ceph-mon[23948]: Updating smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T23:15:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:58 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:58 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:58.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:58 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:58.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:58 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:58.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:58 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:58.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:58 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:15:59.251 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:59 smithi069 ceph-mon[63133]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:15:59.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:59 smithi069 ceph-mon[63133]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:15:59.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:59 smithi069 ceph-mon[63133]: pgmap v9: 97 pgs: 97 active+clean; 326 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.5 MiB/s wr, 583 op/s 2024-04-22T23:15:59.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:59 smithi069 ceph-mon[63133]: Updating smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T23:15:59.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:59 smithi069 ceph-mon[63133]: Updating smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T23:15:59.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:59 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:15:59.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:59 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:15:59.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:15:59 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T23:15:59.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 151968 11.62 MB/sec execute 169 sec latency 43.640 ms 2024-04-22T23:15:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:59 smithi149 ceph-mon[23948]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:15:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:59 smithi149 ceph-mon[23948]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2024-04-22T23:15:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:59 smithi149 ceph-mon[23948]: pgmap v9: 97 pgs: 97 active+clean; 326 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.5 MiB/s wr, 583 op/s 2024-04-22T23:15:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:59 smithi149 ceph-mon[23948]: Updating smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T23:15:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:59 smithi149 ceph-mon[23948]: Updating smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T23:15:59.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:59 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:15:59.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:59 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:15:59.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:15:59 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T23:16:00.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 152390 11.66 MB/sec execute 170 sec latency 31.857 ms 2024-04-22T23:16:01.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 152699 11.64 MB/sec execute 171 sec latency 46.087 ms 2024-04-22T23:16:01.301 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:01 smithi149 ceph-mon[23948]: pgmap v10: 97 pgs: 97 active+clean; 326 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 5.1 MiB/s wr, 534 op/s 2024-04-22T23:16:01.301 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:01 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:01.301 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:01 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:16:01.301 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:01 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T23:16:01.301 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:01 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:01.301 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:01 smithi149 ceph-mon[23948]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:16:01.314 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:01 smithi069 ceph-mon[63133]: pgmap v10: 97 pgs: 97 active+clean; 326 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 5.1 MiB/s wr, 534 op/s 2024-04-22T23:16:01.314 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:01.314 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:16:01.315 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T23:16:01.315 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:01.315 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:16:02.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 153005 11.62 MB/sec execute 172 sec latency 59.386 ms 2024-04-22T23:16:02.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:02 smithi069 ceph-mon[63133]: Upgrade: Updating mon.smithi149 2024-04-22T23:16:02.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:02 smithi069 ceph-mon[63133]: Deploying daemon mon.smithi149 on smithi149 2024-04-22T23:16:02.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:02 smithi149 ceph-mon[23948]: Upgrade: Updating mon.smithi149 2024-04-22T23:16:02.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:02 smithi149 ceph-mon[23948]: Deploying daemon mon.smithi149 on smithi149 2024-04-22T23:16:02.814 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:02 smithi149 systemd[1]: Stopping Ceph mon.smithi149 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:16:03.066 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:02 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149[23944]: 2024-04-22T23:16:02.812+0000 7fbe26f07700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi149 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:16:03.067 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:02 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149[23944]: 2024-04-22T23:16:02.812+0000 7fbe26f07700 -1 mon.smithi149@1(peon) e2 *** Got Signal Terminated *** 2024-04-22T23:16:03.270 INFO:teuthology.orchestra.run.smithi069.stdout: 5 153233 11.57 MB/sec execute 173 sec latency 39.799 ms 2024-04-22T23:16:03.356 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:03 smithi149 podman[49637]: 2024-04-22 23:16:03.066283026 +0000 UTC m=+0.378711157 container died 1a87df28e6506b964a75091ef00e8f08e42cee1501a6feb74360d66989c6d094 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, release=754, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, version=8, distribution-scope=public, vcs-type=git, io.buildah.version=1.19.8, ceph=True, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.openshift.expose-services=, GIT_CLEAN=True, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container) 2024-04-22T23:16:03.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:03 smithi149 podman[49637]: 2024-04-22 23:16:03.489998426 +0000 UTC m=+0.802426561 container cleanup 1a87df28e6506b964a75091ef00e8f08e42cee1501a6feb74360d66989c6d094 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149, version=8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, architecture=x86_64, maintainer=Guillaume Abrioux , release=754, io.openshift.expose-services=, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., RELEASE=HEAD, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870) 2024-04-22T23:16:03.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:03 smithi149 bash[49637]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149 2024-04-22T23:16:04.001 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:03 smithi149 podman[49651]: 2024-04-22 23:16:03.748161901 +0000 UTC m=+0.682812543 container remove 1a87df28e6506b964a75091ef00e8f08e42cee1501a6feb74360d66989c6d094 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, version=8, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream) 2024-04-22T23:16:04.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 153506 11.54 MB/sec execute 174 sec latency 40.531 ms 2024-04-22T23:16:04.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi149.service: Deactivated successfully. 2024-04-22T23:16:04.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 systemd[1]: Stopped Ceph mon.smithi149 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:16:04.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi149.service: Consumed 7.073s CPU time. 2024-04-22T23:16:04.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 systemd[1]: Starting Ceph mon.smithi149 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:16:04.660 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:16:04.812 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 podman[49779]: 2024-04-22 23:16:04.376266383 +0000 UTC m=+0.020753399 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:16:04.812 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 podman[49779]: 2024-04-22 23:16:04.517391906 +0000 UTC m=+0.161878902 container create c208daf7eb1c1761a86b585eb5ba2c0e043792e87c3fe22f37333d1fa93fc692 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240416) 2024-04-22T23:16:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 podman[49779]: 2024-04-22 23:16:04.812132816 +0000 UTC m=+0.456619828 container init c208daf7eb1c1761a86b585eb5ba2c0e043792e87c3fe22f37333d1fa93fc692 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-22T23:16:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 podman[49779]: 2024-04-22 23:16:04.816817905 +0000 UTC m=+0.461304906 container start c208daf7eb1c1761a86b585eb5ba2c0e043792e87c3fe22f37333d1fa93fc692 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-22T23:16:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T23:16:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev), process ceph-mon, pid 2 2024-04-22T23:16:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: pidfile_write: ignore empty --pid-file 2024-04-22T23:16:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: load: jerasure load: lrc 2024-04-22T23:16:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: RocksDB version: 7.9.2 2024-04-22T23:16:05.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Git sha 0 2024-04-22T23:16:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Compile date 2024-04-22 21:33:31 2024-04-22T23:16:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: DB SUMMARY 2024-04-22T23:16:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: DB Session ID: FWVNTC4WOME3SKL021FC 2024-04-22T23:16:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: CURRENT file: CURRENT 2024-04-22T23:16:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T23:16:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: MANIFEST file: MANIFEST-000009 size: 904 Bytes 2024-04-22T23:16:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi149/store.db dir, Total Num: 1, files: 000024.sst 2024-04-22T23:16:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi149/store.db: 000022.log size: 5699575 ; 2024-04-22T23:16:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.error_if_exists: 0 2024-04-22T23:16:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.create_if_missing: 0 2024-04-22T23:16:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.paranoid_checks: 1 2024-04-22T23:16:05.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-22T23:16:05.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T23:16:05.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-22T23:16:05.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.env: 0x55ae74ffccc0 2024-04-22T23:16:05.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.fs: PosixFileSystem 2024-04-22T23:16:05.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.info_log: 0x55ae76379aa0 2024-04-22T23:16:05.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T23:16:05.204 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.statistics: (nil) 2024-04-22T23:16:05.205 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.use_fsync: 0 2024-04-22T23:16:05.205 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_log_file_size: 0 2024-04-22T23:16:05.205 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T23:16:05.205 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T23:16:05.205 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T23:16:05.205 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T23:16:05.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.allow_fallocate: 1 2024-04-22T23:16:05.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T23:16:05.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T23:16:05.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.use_direct_reads: 0 2024-04-22T23:16:05.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T23:16:05.209 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.db_log_dir: 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.wal_dir: 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.write_buffer_manager: 0x55ae7630d0e0 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T23:16:05.210 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T23:16:05.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T23:16:05.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T23:16:05.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T23:16:05.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T23:16:05.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T23:16:05.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.unordered_write: 0 2024-04-22T23:16:05.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T23:16:05.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T23:16:05.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T23:16:05.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T23:16:05.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.row_cache: None 2024-04-22T23:16:05.211 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.wal_filter: None 2024-04-22T23:16:05.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T23:16:05.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T23:16:05.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.two_write_queues: 0 2024-04-22T23:16:05.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T23:16:05.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.wal_compression: 0 2024-04-22T23:16:05.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.atomic_flush: 0 2024-04-22T23:16:05.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T23:16:05.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T23:16:05.212 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T23:16:05.213 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.log_readahead_size: 0 2024-04-22T23:16:05.213 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T23:16:05.213 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T23:16:05.213 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T23:16:05.213 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T23:16:05.213 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T23:16:05.213 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T23:16:05.213 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-22T23:16:05.213 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_background_jobs: 2 2024-04-22T23:16:05.213 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_background_compactions: -1 2024-04-22T23:16:05.213 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_subcompactions: 1 2024-04-22T23:16:05.213 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_open_files: -1 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T23:16:05.214 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_background_flushes: -1 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Compression algorithms supported: 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: kZSTD supported: 0 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: kXpressCompression supported: 0 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: kZlibCompression supported: 1 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: kSnappyCompression supported: 1 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: kLZ4Compression supported: 1 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: kBZip2Compression supported: 0 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi149/store.db/MANIFEST-000009 2024-04-22T23:16:05.215 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.merge_operator: 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_filter: None 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ae76379c80) 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: cache_index_and_filter_blocks: 1 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: pin_top_level_index_and_filter: 1 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: index_type: 0 2024-04-22T23:16:05.216 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: data_block_index_type: 0 2024-04-22T23:16:05.217 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: index_shortening: 1 2024-04-22T23:16:05.217 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T23:16:05.217 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: checksum: 4 2024-04-22T23:16:05.217 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: no_block_cache: 0 2024-04-22T23:16:05.217 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_cache: 0x55ae7633f1f0 2024-04-22T23:16:05.217 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_cache_name: BinnedLRUCache 2024-04-22T23:16:05.217 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_cache_options: 2024-04-22T23:16:05.218 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: capacity : 536870912 2024-04-22T23:16:05.218 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: num_shard_bits : 4 2024-04-22T23:16:05.218 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: strict_capacity_limit : 0 2024-04-22T23:16:05.218 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: high_pri_pool_ratio: 0.000 2024-04-22T23:16:05.218 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_cache_compressed: (nil) 2024-04-22T23:16:05.218 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: persistent_cache: (nil) 2024-04-22T23:16:05.219 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_size: 4096 2024-04-22T23:16:05.219 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_size_deviation: 10 2024-04-22T23:16:05.219 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_restart_interval: 16 2024-04-22T23:16:05.219 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: index_block_restart_interval: 1 2024-04-22T23:16:05.219 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: metadata_block_size: 4096 2024-04-22T23:16:05.219 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: partition_filters: 0 2024-04-22T23:16:05.219 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: use_delta_encoding: 1 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: filter_policy: bloomfilter 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: whole_key_filtering: 1 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: verify_compression: 0 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: read_amp_bytes_per_bit: 0 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: format_version: 5 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: enable_index_compression: 1 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: block_align: 0 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: max_auto_readahead_size: 262144 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: prepopulate_block_cache: 0 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: initial_auto_readahead_size: 8192 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: num_file_reads_for_auto_readahead: 2 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T23:16:05.220 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compression: NoCompression 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.num_levels: 7 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T23:16:05.221 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T23:16:05.222 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T23:16:05.223 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.arena_block_size: 1048576 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T23:16:05.224 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.inplace_update_support: 0 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.bloom_locality: 0 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.max_successive_merges: 0 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.ttl: 2592000 2024-04-22T23:16:05.225 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T23:16:05.226 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-22T23:16:05.226 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-22T23:16:05.226 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.enable_blob_files: false 2024-04-22T23:16:05.226 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.min_blob_size: 0 2024-04-22T23:16:05.226 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T23:16:05.228 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T23:16:05.228 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T23:16:05.228 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T23:16:05.228 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-22T23:16:05.228 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-22T23:16:05.228 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.blob_file_starting_level: 0 2024-04-22T23:16:05.228 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-22T23:16:05.228 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-04-22T23:16:05.228 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T23:16:05.228 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi149/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12233, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T23:16:05.228 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 2cfb8cb6-a00f-42a1-a547-43ee3051dcef 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713827764862009, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713827764877049, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 3507978, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12238, "largest_seqno": 13736, "table_properties": {"data_size": 3501096, "index_size": 4301, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1541, "raw_key_size": 13838, "raw_average_key_size": 23, "raw_value_size": 3487950, "raw_average_value_size": 5901, "num_data_blocks": 196, "num_entries": 591, "num_filter_entries": 591, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1713827764, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "2cfb8cb6-a00f-42a1-a547-43ee3051dcef", "db_session_id": "FWVNTC4WOME3SKL021FC", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713827764877204, "job": 1, "event": "recovery_finished"} 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi149/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ae76454000 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: DB pointer 0x55ae7643e000 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-22T23:16:05.229 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ** DB Stats ** 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ** Compaction Stats [default] ** 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: L0 1/0 3.35 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 547.1 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: L6 1/0 7.56 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-22T23:16:05.230 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Sum 2/0 10.90 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 547.1 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 547.1 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ** Compaction Stats [default] ** 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 547.1 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T23:16:05.231 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Cumulative compaction: 0.00 GB write, 106.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Interval compaction: 0.00 GB write, 106.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Block cache BinnedLRUCache@0x55ae7633f1f0#2 capacity: 512.00 MB usage: 6.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: starting mon.smithi149 rank 1 at public addrs [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] at bind addrs [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi149 fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: mon.smithi149@-1(???) e2 preinit fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: mon.smithi149@-1(???).mds e7 new map 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: mon.smithi149@-1(???).mds e7 print_map 2024-04-22T23:16:05.232 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: e7 2024-04-22T23:16:05.233 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T23:16:05.233 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T23:16:05.233 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: legacy client fscid: 1 2024-04-22T23:16:05.233 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-04-22T23:16:05.233 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Filesystem 'foofs' (1) 2024-04-22T23:16:05.233 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: fs_name foofs 2024-04-22T23:16:05.233 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: epoch 5 2024-04-22T23:16:05.233 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-22T23:16:05.233 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: created 2024-04-22T23:10:32.113295+0000 2024-04-22T23:16:05.233 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: modified 2024-04-22T23:10:38.133401+0000 2024-04-22T23:16:05.233 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: tableserver 0 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: root 0 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: session_timeout 60 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: session_autoclose 300 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: max_file_size 1099511627776 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: max_xattr_size 65536 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: required_client_features {} 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: last_failure 0 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: last_failure_osd_epoch 0 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: max_mds 1 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: in 0 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: up {0=14498} 2024-04-22T23:16:05.234 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: failed 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: damaged 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: stopped 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: data_pools [3] 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: metadata_pool 2 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: inline_data disabled 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: balancer 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: bal_rank_mask -1 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: standby_count_wanted 1 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: qdb_cluster leader: 0 members: 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: [mds.foofs.smithi069.jiyres{0:14498} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.69:6834/1879302209,v1:172.21.15.69:6835/1879302209] compat {c=[1],r=[1],i=[7ff]}] 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: Standby daemons: 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: 2024-04-22T23:16:05.235 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout: [mds.foofs.smithi149.ickhha{-1:24251} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.149:6832/1034958666,v1:172.21.15.149:6833/1034958666] compat {c=[1],r=[1],i=[7ff]}] 2024-04-22T23:16:05.236 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: mon.smithi149@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-22T23:16:05.236 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: mon.smithi149@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T23:16:05.236 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: mon.smithi149@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T23:16:05.236 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: mon.smithi149@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T23:16:05.236 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 ceph-mon[49797]: mon.smithi149@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-04-22T23:16:05.236 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 bash[49779]: c208daf7eb1c1761a86b585eb5ba2c0e043792e87c3fe22f37333d1fa93fc692 2024-04-22T23:16:05.236 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:04 smithi149 systemd[1]: Started Ceph mon.smithi149 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:16:05.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 153858 11.51 MB/sec execute 175 sec latency 53.966 ms 2024-04-22T23:16:05.597 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:16:05.598 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (2m) 10s ago 11m 23.7M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:16:05.598 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (11m) 10s ago 11m 6400k - 17.2.0 e1d6a67b021e f29a69890fd2 2024-04-22T23:16:05.598 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (10m) 14s ago 10m 6400k - 17.2.0 e1d6a67b021e d3576856bdfd 2024-04-22T23:16:05.598 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (2m) 10s ago 11m 43.5M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:16:05.598 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (5m) 10s ago 5m 38.3M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:16:05.598 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (5m) 14s ago 5m 14.0M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:16:05.598 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (106s) 10s ago 12m 477M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:16:05.598 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (51s) 14s ago 10m 448M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:16:05.598 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (31s) 10s ago 12m 31.3M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:16:05.598 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 starting - - - 2048M 2024-04-22T23:16:05.598 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (5m) 10s ago 5m 250M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:16:05.598 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (5m) 14s ago 5m 85.9M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:16:05.599 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (2m) 10s ago 11m 9097k - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:16:05.599 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (114s) 14s ago 10m 15.7M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:16:05.599 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (9m) 10s ago 9m 506M 1936M 17.2.0 e1d6a67b021e 777db7a3d809 2024-04-22T23:16:05.599 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (9m) 10s ago 9m 445M 1936M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:16:05.599 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (8m) 10s ago 8m 333M 1936M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:16:05.599 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (8m) 10s ago 8m 467M 1936M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:16:05.599 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (8m) 14s ago 8m 482M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:16:05.599 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (7m) 14s ago 7m 565M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:16:05.599 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (7m) 14s ago 7m 470M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:16:05.599 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (6m) 14s ago 6m 374M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:16:05.599 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (58s) 10s ago 10m 61.8M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:16:06.178 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:16:06.178 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:16:06.178 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T23:16:06.178 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 1 2024-04-22T23:16:06.178 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 3 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:16:06.179 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:16:06.244 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:06 smithi069 ceph-mon[63133]: pgmap v11: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 7.1 MiB/s wr, 681 op/s 2024-04-22T23:16:06.272 INFO:teuthology.orchestra.run.smithi069.stdout: 5 154300 11.49 MB/sec execute 176 sec latency 103.787 ms 2024-04-22T23:16:06.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:06 smithi149 ceph-mon[49797]: pgmap v11: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 7.1 MiB/s wr, 681 op/s 2024-04-22T23:16:06.725 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:16:06.725 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:16:06.725 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:16:06.725 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:16:06.725 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:16:06.726 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-04-22T23:16:06.726 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:16:06.726 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "3/23 daemons upgraded", 2024-04-22T23:16:06.726 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading mon daemons", 2024-04-22T23:16:06.726 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:16:06.726 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:16:07.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 154859 11.51 MB/sec execute 177 sec latency 41.307 ms 2024-04-22T23:16:07.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:16:07.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 466 op/s 2024-04-22T23:16:07.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:16:07.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:16:07.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: mon.smithi069 calling monitor election 2024-04-22T23:16:07.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: mon.smithi149 calling monitor election 2024-04-22T23:16:07.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: mon.smithi069 is new leader, mons smithi069,smithi149 in quorum (ranks 0,1) 2024-04-22T23:16:07.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: monmap epoch 3 2024-04-22T23:16:07.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:16:07.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: last_changed 2024-04-22T23:16:06.202484+0000 2024-04-22T23:16:07.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: created 2024-04-22T23:03:14.830052+0000 2024-04-22T23:16:07.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: min_mon_release 19 (squid) 2024-04-22T23:16:07.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: election_strategy: 1 2024-04-22T23:16:07.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: 0: [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon.smithi069 2024-04-22T23:16:07.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: 1: [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] mon.smithi149 2024-04-22T23:16:07.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 1 up:standby 2024-04-22T23:16:07.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T23:16:07.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: mgrmap e40: smithi069.lyiiqg(active, since 20s), standbys: smithi149.rxqkqk 2024-04-22T23:16:07.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: overall HEALTH_OK 2024-04-22T23:16:07.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:07.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:07.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:07 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:16:07.322 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:16:07.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 466 op/s 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mon metadata", "id": "smithi149"}]: dispatch 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: mon.smithi069 calling monitor election 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: mon.smithi149 calling monitor election 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: mon.smithi069 is new leader, mons smithi069,smithi149 in quorum (ranks 0,1) 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: monmap epoch 3 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: last_changed 2024-04-22T23:16:06.202484+0000 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: created 2024-04-22T23:03:14.830052+0000 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: min_mon_release 19 (squid) 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: election_strategy: 1 2024-04-22T23:16:07.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: 0: [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon.smithi069 2024-04-22T23:16:07.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: 1: [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] mon.smithi149 2024-04-22T23:16:07.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 1 up:standby 2024-04-22T23:16:07.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T23:16:07.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: mgrmap e40: smithi069.lyiiqg(active, since 20s), standbys: smithi149.rxqkqk 2024-04-22T23:16:07.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: overall HEALTH_OK 2024-04-22T23:16:07.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:07.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:07.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:07 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:16:08.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 155491 11.56 MB/sec execute 178 sec latency 29.383 ms 2024-04-22T23:16:08.280 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:08 smithi069 ceph-mon[63133]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:16:08.280 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:08 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/1982451944' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:16:08.572 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:08 smithi149 ceph-mon[49797]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:16:08.572 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:08 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/1982451944' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:16:09.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 156048 11.61 MB/sec execute 179 sec latency 28.992 ms 2024-04-22T23:16:09.284 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:09 smithi069 ceph-mon[63133]: pgmap v14: 97 pgs: 97 active+clean; 324 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 690 op/s 2024-04-22T23:16:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:09 smithi149 ceph-mon[49797]: pgmap v14: 97 pgs: 97 active+clean; 324 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 690 op/s 2024-04-22T23:16:10.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 156469 11.63 MB/sec execute 180 sec latency 36.103 ms 2024-04-22T23:16:11.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:11 smithi069 ceph-mon[63133]: pgmap v15: 97 pgs: 97 active+clean; 324 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 450 op/s 2024-04-22T23:16:11.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 156749 11.59 MB/sec execute 181 sec latency 45.674 ms 2024-04-22T23:16:11.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:11 smithi149 ceph-mon[49797]: pgmap v15: 97 pgs: 97 active+clean; 324 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 450 op/s 2024-04-22T23:16:12.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 157122 11.56 MB/sec execute 182 sec latency 41.787 ms 2024-04-22T23:16:12.780 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:12 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:12.780 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:12 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:12.780 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:12 smithi069 ceph-mon[63133]: pgmap v16: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.1 MiB/s wr, 544 op/s 2024-04-22T23:16:12.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:12 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:12.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:12 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:12.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:12 smithi149 ceph-mon[49797]: pgmap v16: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.1 MiB/s wr, 544 op/s 2024-04-22T23:16:13.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 157558 11.54 MB/sec execute 183 sec latency 44.091 ms 2024-04-22T23:16:13.971 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:13.972 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:14.043 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:14.043 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:14.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 158036 11.55 MB/sec execute 184 sec latency 51.556 ms 2024-04-22T23:16:15.065 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:14 smithi069 ceph-mon[63133]: pgmap v17: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.7 MiB/s wr, 579 op/s 2024-04-22T23:16:15.091 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:14 smithi149 ceph-mon[49797]: pgmap v17: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.7 MiB/s wr, 579 op/s 2024-04-22T23:16:15.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 158541 11.54 MB/sec execute 185 sec latency 56.983 ms 2024-04-22T23:16:16.085 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:15 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:16:16.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:15 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:16:16.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 158889 11.55 MB/sec execute 186 sec latency 32.931 ms 2024-04-22T23:16:16.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:16 smithi069 ceph-mon[63133]: pgmap v18: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 499 op/s 2024-04-22T23:16:16.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:16.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:16.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:16.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:16:16.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:16.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:16.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:16:16.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T23:16:16.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:17.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 159189 11.55 MB/sec execute 187 sec latency 38.184 ms 2024-04-22T23:16:17.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:16 smithi149 ceph-mon[49797]: pgmap v18: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 499 op/s 2024-04-22T23:16:17.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:17.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:17.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:17.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:16:17.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:17.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:17.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:16:17.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T23:16:17.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:18.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:17 smithi069 ceph-mon[63133]: Reconfiguring mon.smithi069 (monmap changed)... 2024-04-22T23:16:18.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:17 smithi069 ceph-mon[63133]: Reconfiguring daemon mon.smithi069 on smithi069 2024-04-22T23:16:18.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 159504 11.56 MB/sec execute 188 sec latency 51.166 ms 2024-04-22T23:16:18.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:17 smithi149 ceph-mon[49797]: Reconfiguring mon.smithi069 (monmap changed)... 2024-04-22T23:16:18.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:17 smithi149 ceph-mon[49797]: Reconfiguring daemon mon.smithi069 on smithi069 2024-04-22T23:16:19.166 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:18 smithi069 ceph-mon[63133]: pgmap v19: 97 pgs: 97 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 761 op/s 2024-04-22T23:16:19.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 159938 11.58 MB/sec execute 189 sec latency 43.170 ms 2024-04-22T23:16:19.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:18 smithi149 ceph-mon[49797]: pgmap v19: 97 pgs: 97 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 761 op/s 2024-04-22T23:16:20.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 160323 11.55 MB/sec execute 190 sec latency 21.877 ms 2024-04-22T23:16:21.179 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:20 smithi069 ceph-mon[63133]: pgmap v20: 97 pgs: 97 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.6 MiB/s wr, 537 op/s 2024-04-22T23:16:21.179 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:21.179 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:21.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:20 smithi069 ceph-mon[63133]: Reconfiguring mgr.smithi069.lyiiqg (monmap changed)... 2024-04-22T23:16:21.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.lyiiqg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:16:21.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:16:21.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:21.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:20 smithi069 ceph-mon[63133]: Reconfiguring daemon mgr.smithi069.lyiiqg on smithi069 2024-04-22T23:16:21.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 160845 11.54 MB/sec execute 191 sec latency 32.846 ms 2024-04-22T23:16:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:20 smithi149 ceph-mon[49797]: pgmap v20: 97 pgs: 97 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.6 MiB/s wr, 537 op/s 2024-04-22T23:16:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:20 smithi149 ceph-mon[49797]: Reconfiguring mgr.smithi069.lyiiqg (monmap changed)... 2024-04-22T23:16:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.lyiiqg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:16:21.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:16:21.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:21.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:20 smithi149 ceph-mon[49797]: Reconfiguring daemon mgr.smithi069.lyiiqg on smithi069 2024-04-22T23:16:22.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 161344 11.53 MB/sec execute 192 sec latency 71.551 ms 2024-04-22T23:16:23.187 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:22 smithi069 ceph-mon[63133]: pgmap v21: 97 pgs: 97 active+clean; 321 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 7.5 MiB/s wr, 700 op/s 2024-04-22T23:16:23.271 INFO:teuthology.orchestra.run.smithi069.stdout: 5 161990 11.55 MB/sec execute 193 sec latency 50.726 ms 2024-04-22T23:16:23.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:22 smithi149 ceph-mon[49797]: pgmap v21: 97 pgs: 97 active+clean; 321 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 7.5 MiB/s wr, 700 op/s 2024-04-22T23:16:24.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 162459 11.56 MB/sec execute 194 sec latency 66.055 ms 2024-04-22T23:16:24.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:24 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:24.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:24 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:24.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:24 smithi069 ceph-mon[63133]: Reconfiguring crash.smithi069 (monmap changed)... 2024-04-22T23:16:24.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:24 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:16:24.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:24 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:24.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:24 smithi069 ceph-mon[63133]: Reconfiguring daemon crash.smithi069 on smithi069 2024-04-22T23:16:24.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:24 smithi069 ceph-mon[63133]: pgmap v22: 97 pgs: 97 active+clean; 321 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 695 op/s 2024-04-22T23:16:24.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:24 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:24.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:24 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:24.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:24 smithi149 ceph-mon[49797]: Reconfiguring crash.smithi069 (monmap changed)... 2024-04-22T23:16:24.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:24 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:16:24.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:24 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:24.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:24 smithi149 ceph-mon[49797]: Reconfiguring daemon crash.smithi069 on smithi069 2024-04-22T23:16:24.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:24 smithi149 ceph-mon[49797]: pgmap v22: 97 pgs: 97 active+clean; 321 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 695 op/s 2024-04-22T23:16:25.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 162976 11.62 MB/sec execute 195 sec latency 29.054 ms 2024-04-22T23:16:26.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 163383 11.63 MB/sec execute 196 sec latency 38.111 ms 2024-04-22T23:16:26.788 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:26 smithi069 ceph-mon[63133]: pgmap v23: 97 pgs: 97 active+clean; 321 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.9 MiB/s wr, 513 op/s 2024-04-22T23:16:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:26 smithi149 ceph-mon[49797]: pgmap v23: 97 pgs: 97 active+clean; 321 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.9 MiB/s wr, 513 op/s 2024-04-22T23:16:27.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 163653 11.61 MB/sec execute 197 sec latency 38.667 ms 2024-04-22T23:16:27.804 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:27 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:27.805 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:27 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:27.805 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:27 smithi069 ceph-mon[63133]: Reconfiguring osd.0 (monmap changed)... 2024-04-22T23:16:27.805 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:27 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T23:16:27.805 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:27 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:27.805 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:27 smithi069 ceph-mon[63133]: Reconfiguring daemon osd.0 on smithi069 2024-04-22T23:16:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:27 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:27 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:27 smithi149 ceph-mon[49797]: Reconfiguring osd.0 (monmap changed)... 2024-04-22T23:16:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:27 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T23:16:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:27 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:27 smithi149 ceph-mon[49797]: Reconfiguring daemon osd.0 on smithi069 2024-04-22T23:16:28.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 163904 11.56 MB/sec execute 198 sec latency 63.285 ms 2024-04-22T23:16:28.821 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:28 smithi069 ceph-mon[63133]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 794 op/s 2024-04-22T23:16:29.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:28 smithi149 ceph-mon[49797]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 794 op/s 2024-04-22T23:16:29.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 164191 11.53 MB/sec execute 199 sec latency 61.421 ms 2024-04-22T23:16:30.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:30.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:30.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:30 smithi069 ceph-mon[63133]: Reconfiguring osd.1 (monmap changed)... 2024-04-22T23:16:30.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T23:16:30.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:30.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:30 smithi069 ceph-mon[63133]: Reconfiguring daemon osd.1 on smithi069 2024-04-22T23:16:30.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 164494 11.50 MB/sec execute 200 sec latency 40.546 ms 2024-04-22T23:16:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:30 smithi149 ceph-mon[49797]: Reconfiguring osd.1 (monmap changed)... 2024-04-22T23:16:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T23:16:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:30.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:30 smithi149 ceph-mon[49797]: Reconfiguring daemon osd.1 on smithi069 2024-04-22T23:16:31.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:31 smithi069 ceph-mon[63133]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 532 op/s 2024-04-22T23:16:31.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:16:31.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 164739 11.48 MB/sec execute 201 sec latency 60.915 ms 2024-04-22T23:16:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:31 smithi149 ceph-mon[49797]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 532 op/s 2024-04-22T23:16:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:16:32.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 165123 11.47 MB/sec execute 202 sec latency 59.527 ms 2024-04-22T23:16:32.893 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:32 smithi069 ceph-mon[63133]: pgmap v26: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 708 op/s 2024-04-22T23:16:32.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:32 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:32.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:32 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:32.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:32 smithi069 ceph-mon[63133]: Reconfiguring osd.2 (monmap changed)... 2024-04-22T23:16:32.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:32 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T23:16:32.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:32 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:32.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:32 smithi069 ceph-mon[63133]: Reconfiguring daemon osd.2 on smithi069 2024-04-22T23:16:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:32 smithi149 ceph-mon[49797]: pgmap v26: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 708 op/s 2024-04-22T23:16:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:32 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:32 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:32 smithi149 ceph-mon[49797]: Reconfiguring osd.2 (monmap changed)... 2024-04-22T23:16:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:32 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T23:16:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:32 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:33.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:32 smithi149 ceph-mon[49797]: Reconfiguring daemon osd.2 on smithi069 2024-04-22T23:16:33.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 165589 11.46 MB/sec execute 203 sec latency 41.154 ms 2024-04-22T23:16:34.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 166062 11.48 MB/sec execute 204 sec latency 29.734 ms 2024-04-22T23:16:34.936 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:34 smithi069 ceph-mon[63133]: pgmap v27: 97 pgs: 97 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.4 MiB/s wr, 558 op/s 2024-04-22T23:16:34.937 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:34 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:34.937 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:34 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:34.937 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:34 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T23:16:34.937 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:34 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:34 smithi149 ceph-mon[49797]: pgmap v27: 97 pgs: 97 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.4 MiB/s wr, 558 op/s 2024-04-22T23:16:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:34 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:34 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:34 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T23:16:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:34 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:35.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 166518 11.52 MB/sec execute 205 sec latency 31.181 ms 2024-04-22T23:16:35.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:35 smithi069 ceph-mon[63133]: Reconfiguring osd.3 (monmap changed)... 2024-04-22T23:16:35.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:35 smithi069 ceph-mon[63133]: Reconfiguring daemon osd.3 on smithi069 2024-04-22T23:16:36.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:35 smithi149 ceph-mon[49797]: Reconfiguring osd.3 (monmap changed)... 2024-04-22T23:16:36.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:35 smithi149 ceph-mon[49797]: Reconfiguring daemon osd.3 on smithi069 2024-04-22T23:16:36.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 167014 11.54 MB/sec execute 206 sec latency 29.314 ms 2024-04-22T23:16:36.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:36 smithi149 ceph-mon[49797]: pgmap v28: 97 pgs: 97 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 469 op/s 2024-04-22T23:16:36.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:36 smithi069 ceph-mon[63133]: pgmap v28: 97 pgs: 97 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 469 op/s 2024-04-22T23:16:37.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 167384 11.52 MB/sec execute 207 sec latency 32.502 ms 2024-04-22T23:16:37.882 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:16:38.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 167762 11.49 MB/sec execute 208 sec latency 38.378 ms 2024-04-22T23:16:38.640 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:38.640 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:38.640 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:38 smithi069 ceph-mon[63133]: Reconfiguring mds.foofs.smithi069.jiyres (monmap changed)... 2024-04-22T23:16:38.640 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.jiyres", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T23:16:38.640 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:38.640 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:38 smithi069 ceph-mon[63133]: Reconfiguring daemon mds.foofs.smithi069.jiyres on smithi069 2024-04-22T23:16:38.640 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:38 smithi069 ceph-mon[63133]: pgmap v29: 97 pgs: 97 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.7 MiB/s wr, 731 op/s 2024-04-22T23:16:38.640 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:38 smithi069 ceph-mon[63133]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:16:38.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:38.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:38.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:38 smithi149 ceph-mon[49797]: Reconfiguring mds.foofs.smithi069.jiyres (monmap changed)... 2024-04-22T23:16:38.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.jiyres", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T23:16:38.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:38.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:38 smithi149 ceph-mon[49797]: Reconfiguring daemon mds.foofs.smithi069.jiyres on smithi069 2024-04-22T23:16:38.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:38 smithi149 ceph-mon[49797]: pgmap v29: 97 pgs: 97 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.7 MiB/s wr, 731 op/s 2024-04-22T23:16:38.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:38 smithi149 ceph-mon[49797]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:16:38.965 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:16:38.965 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (2m) 43s ago 11m 23.7M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:16:38.965 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (11m) 43s ago 11m 6400k - 17.2.0 e1d6a67b021e f29a69890fd2 2024-04-22T23:16:38.965 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (11m) 27s ago 11m 6400k - 17.2.0 e1d6a67b021e d3576856bdfd 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (2m) 43s ago 11m 43.5M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (6m) 43s ago 6m 38.3M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (6m) 27s ago 6m 14.4M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (2m) 43s ago 13m 477M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (85s) 27s ago 11m 449M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (64s) 43s ago 13m 31.3M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (34s) 27s ago 11m 21.8M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (5m) 43s ago 5m 250M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (5m) 27s ago 5m 86.4M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (2m) 43s ago 11m 9097k - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (2m) 27s ago 11m 16.6M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (10m) 43s ago 10m 506M 1936M 17.2.0 e1d6a67b021e 777db7a3d809 2024-04-22T23:16:38.966 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (9m) 43s ago 9m 445M 1936M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:16:38.967 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (9m) 43s ago 9m 333M 1936M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:16:38.967 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (9m) 43s ago 9m 467M 1936M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:16:38.967 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (8m) 27s ago 8m 503M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:16:38.967 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (8m) 27s ago 8m 324M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:16:38.967 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (7m) 27s ago 7m 509M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:16:38.967 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (7m) 27s ago 7m 393M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:16:38.967 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (92s) 43s ago 10m 61.8M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:16:39.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 168146 11.47 MB/sec execute 209 sec latency 41.981 ms 2024-04-22T23:16:39.583 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:16:39.583 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:16:39.583 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-22T23:16:39.584 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 4 2024-04-22T23:16:39.585 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:16:39.585 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:16:39.827 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:39 smithi069 ceph-mon[63133]: from='client.44103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:16:39.827 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:39 smithi069 ceph-mon[63133]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:16:39.827 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:39 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/200173994' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:16:39.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:39 smithi149 ceph-mon[49797]: from='client.44103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:16:39.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:39 smithi149 ceph-mon[49797]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:16:39.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:39 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/200173994' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:16:40.078 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:16:40.078 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:16:40.078 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:16:40.079 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:16:40.079 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:16:40.079 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:16:40.079 INFO:teuthology.orchestra.run.smithi069.stdout: "mon" 2024-04-22T23:16:40.079 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:16:40.079 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "4/23 daemons upgraded", 2024-04-22T23:16:40.079 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading mon daemons", 2024-04-22T23:16:40.079 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:16:40.079 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:16:40.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 168454 11.45 MB/sec execute 210 sec latency 46.019 ms 2024-04-22T23:16:40.648 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:16:41.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 168861 11.46 MB/sec execute 211 sec latency 42.667 ms 2024-04-22T23:16:41.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:41 smithi069 ceph-mon[63133]: pgmap v30: 97 pgs: 97 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 449 op/s 2024-04-22T23:16:41.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:41 smithi069 ceph-mon[63133]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:16:41.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:41 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:41.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:41 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:41.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:41 smithi069 ceph-mon[63133]: Reconfiguring crash.smithi149 (monmap changed)... 2024-04-22T23:16:41.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:41 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:16:41.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:41 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:41.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:41 smithi069 ceph-mon[63133]: Reconfiguring daemon crash.smithi149 on smithi149 2024-04-22T23:16:41.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:41 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/2086239549' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:16:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:41 smithi149 ceph-mon[49797]: pgmap v30: 97 pgs: 97 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 449 op/s 2024-04-22T23:16:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:41 smithi149 ceph-mon[49797]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:16:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:41 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:41.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:41 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:41.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:41 smithi149 ceph-mon[49797]: Reconfiguring crash.smithi149 (monmap changed)... 2024-04-22T23:16:41.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:41 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:16:41.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:41 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:41.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:41 smithi149 ceph-mon[49797]: Reconfiguring daemon crash.smithi149 on smithi149 2024-04-22T23:16:41.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:41 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/2086239549' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:16:42.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 169302 11.45 MB/sec execute 212 sec latency 48.901 ms 2024-04-22T23:16:43.170 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:43 smithi069 ceph-mon[63133]: pgmap v31: 97 pgs: 97 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.2 MiB/s wr, 622 op/s 2024-04-22T23:16:43.171 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:43.171 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:43.171 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.rxqkqk", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:16:43.171 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:16:43.171 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:43.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 169631 11.45 MB/sec execute 213 sec latency 45.326 ms 2024-04-22T23:16:43.344 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:43 smithi149 ceph-mon[49797]: pgmap v31: 97 pgs: 97 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.2 MiB/s wr, 622 op/s 2024-04-22T23:16:43.344 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:43.344 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:43.345 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi149.rxqkqk", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T23:16:43.345 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T23:16:43.345 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:44.186 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:44 smithi069 ceph-mon[63133]: Reconfiguring mgr.smithi149.rxqkqk (monmap changed)... 2024-04-22T23:16:44.186 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:44 smithi069 ceph-mon[63133]: Reconfiguring daemon mgr.smithi149.rxqkqk on smithi149 2024-04-22T23:16:44.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 169979 11.46 MB/sec execute 214 sec latency 49.172 ms 2024-04-22T23:16:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:44 smithi149 ceph-mon[49797]: Reconfiguring mgr.smithi149.rxqkqk (monmap changed)... 2024-04-22T23:16:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:44 smithi149 ceph-mon[49797]: Reconfiguring daemon mgr.smithi149.rxqkqk on smithi149 2024-04-22T23:16:45.205 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:45 smithi069 ceph-mon[63133]: pgmap v32: 97 pgs: 97 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 511 op/s 2024-04-22T23:16:45.205 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:45.205 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:45.206 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:45 smithi069 ceph-mon[63133]: Reconfiguring mon.smithi149 (monmap changed)... 2024-04-22T23:16:45.206 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:16:45.206 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T23:16:45.206 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:45.206 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:45 smithi069 ceph-mon[63133]: Reconfiguring daemon mon.smithi149 on smithi149 2024-04-22T23:16:45.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 170377 11.47 MB/sec execute 215 sec latency 51.624 ms 2024-04-22T23:16:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:45 smithi149 ceph-mon[49797]: pgmap v32: 97 pgs: 97 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 511 op/s 2024-04-22T23:16:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:45 smithi149 ceph-mon[49797]: Reconfiguring mon.smithi149 (monmap changed)... 2024-04-22T23:16:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T23:16:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T23:16:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:45 smithi149 ceph-mon[49797]: Reconfiguring daemon mon.smithi149 on smithi149 2024-04-22T23:16:46.215 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:16:46.215 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:46.216 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:46.216 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T23:16:46.216 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:46.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 170636 11.46 MB/sec execute 216 sec latency 46.507 ms 2024-04-22T23:16:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:16:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T23:16:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:47.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:47 smithi069 ceph-mon[63133]: pgmap v33: 97 pgs: 97 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 498 op/s 2024-04-22T23:16:47.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:47 smithi069 ceph-mon[63133]: Reconfiguring osd.4 (monmap changed)... 2024-04-22T23:16:47.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:47 smithi069 ceph-mon[63133]: Reconfiguring daemon osd.4 on smithi149 2024-04-22T23:16:47.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 170877 11.43 MB/sec execute 217 sec latency 53.429 ms 2024-04-22T23:16:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:47 smithi149 ceph-mon[49797]: pgmap v33: 97 pgs: 97 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 498 op/s 2024-04-22T23:16:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:47 smithi149 ceph-mon[49797]: Reconfiguring osd.4 (monmap changed)... 2024-04-22T23:16:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:47 smithi149 ceph-mon[49797]: Reconfiguring daemon osd.4 on smithi149 2024-04-22T23:16:48.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 171127 11.40 MB/sec execute 218 sec latency 85.375 ms 2024-04-22T23:16:49.075 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:48 smithi069 ceph-mon[63133]: pgmap v34: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.4 MiB/s wr, 700 op/s 2024-04-22T23:16:49.075 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:48 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:49.075 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:48 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:49.075 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:48 smithi069 ceph-mon[63133]: Reconfiguring osd.5 (monmap changed)... 2024-04-22T23:16:49.075 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:48 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T23:16:49.075 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:48 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:49.075 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:48 smithi069 ceph-mon[63133]: Reconfiguring daemon osd.5 on smithi149 2024-04-22T23:16:49.277 INFO:teuthology.orchestra.run.smithi069.stdout: 5 171408 11.36 MB/sec execute 219 sec latency 88.326 ms 2024-04-22T23:16:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:48 smithi149 ceph-mon[49797]: pgmap v34: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.4 MiB/s wr, 700 op/s 2024-04-22T23:16:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:48 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:48 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:48 smithi149 ceph-mon[49797]: Reconfiguring osd.5 (monmap changed)... 2024-04-22T23:16:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:48 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T23:16:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:48 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:49.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:48 smithi149 ceph-mon[49797]: Reconfiguring daemon osd.5 on smithi149 2024-04-22T23:16:50.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 171655 11.33 MB/sec execute 220 sec latency 50.931 ms 2024-04-22T23:16:51.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:50 smithi069 ceph-mon[63133]: pgmap v35: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 439 op/s 2024-04-22T23:16:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:50 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:50 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:50 smithi069 ceph-mon[63133]: Reconfiguring osd.6 (monmap changed)... 2024-04-22T23:16:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:50 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T23:16:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:50 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:50 smithi069 ceph-mon[63133]: Reconfiguring daemon osd.6 on smithi149 2024-04-22T23:16:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:50 smithi149 ceph-mon[49797]: pgmap v35: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 439 op/s 2024-04-22T23:16:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:50 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:50 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:50 smithi149 ceph-mon[49797]: Reconfiguring osd.6 (monmap changed)... 2024-04-22T23:16:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:50 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T23:16:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:50 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:50 smithi149 ceph-mon[49797]: Reconfiguring daemon osd.6 on smithi149 2024-04-22T23:16:51.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 171943 11.31 MB/sec execute 221 sec latency 45.186 ms 2024-04-22T23:16:52.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 172356 11.32 MB/sec execute 222 sec latency 44.983 ms 2024-04-22T23:16:52.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:52 smithi069 ceph-mon[63133]: pgmap v36: 97 pgs: 97 active+clean; 319 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 625 op/s 2024-04-22T23:16:52.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:52 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:52.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:52 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:52.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:52 smithi069 ceph-mon[63133]: Reconfiguring osd.7 (monmap changed)... 2024-04-22T23:16:52.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:52 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T23:16:52.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:52 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:52.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:52 smithi069 ceph-mon[63133]: Reconfiguring daemon osd.7 on smithi149 2024-04-22T23:16:53.027 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:52 smithi149 ceph-mon[49797]: pgmap v36: 97 pgs: 97 active+clean; 319 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 625 op/s 2024-04-22T23:16:53.027 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:52 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:53.027 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:52 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:53.027 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:52 smithi149 ceph-mon[49797]: Reconfiguring osd.7 (monmap changed)... 2024-04-22T23:16:53.027 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:52 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T23:16:53.027 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:52 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:53.027 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:52 smithi149 ceph-mon[49797]: Reconfiguring daemon osd.7 on smithi149 2024-04-22T23:16:53.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 172726 11.29 MB/sec execute 223 sec latency 44.493 ms 2024-04-22T23:16:54.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 173059 11.29 MB/sec execute 224 sec latency 48.020 ms 2024-04-22T23:16:54.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:54 smithi069 ceph-mon[63133]: pgmap v37: 97 pgs: 97 active+clean; 319 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.3 MiB/s wr, 453 op/s 2024-04-22T23:16:54.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:54 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:54.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:54 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:54.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:54 smithi069 ceph-mon[63133]: Reconfiguring mds.foofs.smithi149.ickhha (monmap changed)... 2024-04-22T23:16:54.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:54 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.ickhha", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T23:16:54.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:54 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:54.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:54 smithi069 ceph-mon[63133]: Reconfiguring daemon mds.foofs.smithi149.ickhha on smithi149 2024-04-22T23:16:55.148 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:54 smithi149 ceph-mon[49797]: pgmap v37: 97 pgs: 97 active+clean; 319 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.3 MiB/s wr, 453 op/s 2024-04-22T23:16:55.148 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:54 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:55.148 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:54 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:55.148 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:54 smithi149 ceph-mon[49797]: Reconfiguring mds.foofs.smithi149.ickhha (monmap changed)... 2024-04-22T23:16:55.148 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:54 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.ickhha", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T23:16:55.149 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:54 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:55.149 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:54 smithi149 ceph-mon[49797]: Reconfiguring daemon mds.foofs.smithi149.ickhha on smithi149 2024-04-22T23:16:55.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 173292 11.28 MB/sec execute 225 sec latency 49.304 ms 2024-04-22T23:16:56.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 173537 11.28 MB/sec execute 226 sec latency 93.116 ms 2024-04-22T23:16:57.028 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:56 smithi069 ceph-mon[63133]: pgmap v38: 97 pgs: 97 active+clean; 319 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 388 op/s 2024-04-22T23:16:57.028 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:57.029 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:57.029 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:16:57.029 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:16:57.029 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:16:57.029 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:56 smithi069 ceph-mon[63133]: Upgrade: Setting container_image for all mon 2024-04-22T23:16:57.029 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:57.029 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi069"}]: dispatch 2024-04-22T23:16:57.029 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi069"}]': finished 2024-04-22T23:16:57.029 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi149"}]: dispatch 2024-04-22T23:16:57.029 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi149"}]': finished 2024-04-22T23:16:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:56 smithi149 ceph-mon[49797]: pgmap v38: 97 pgs: 97 active+clean; 319 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 388 op/s 2024-04-22T23:16:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:16:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:16:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:16:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:56 smithi149 ceph-mon[49797]: Upgrade: Setting container_image for all mon 2024-04-22T23:16:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi069"}]: dispatch 2024-04-22T23:16:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi069"}]': finished 2024-04-22T23:16:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi149"}]: dispatch 2024-04-22T23:16:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi149"}]': finished 2024-04-22T23:16:57.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 173846 11.28 MB/sec execute 227 sec latency 83.869 ms 2024-04-22T23:16:58.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 174091 11.26 MB/sec execute 228 sec latency 52.165 ms 2024-04-22T23:16:59.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 174373 11.24 MB/sec execute 229 sec latency 46.498 ms 2024-04-22T23:16:59.331 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:59 smithi069 ceph-mon[63133]: pgmap v39: 97 pgs: 97 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.9 MiB/s wr, 563 op/s 2024-04-22T23:16:59.331 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:59 smithi069 ceph-mon[63133]: Upgrade: Updating crash.smithi069 (1/2) 2024-04-22T23:16:59.332 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:59 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:59.332 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:59 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:16:59.332 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:59 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:59.332 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:16:59 smithi069 ceph-mon[63133]: Deploying daemon crash.smithi069 on smithi069 2024-04-22T23:16:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:59 smithi149 ceph-mon[49797]: pgmap v39: 97 pgs: 97 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.9 MiB/s wr, 563 op/s 2024-04-22T23:16:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:59 smithi149 ceph-mon[49797]: Upgrade: Updating crash.smithi069 (1/2) 2024-04-22T23:16:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:59 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:16:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:59 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:16:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:59 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:16:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:16:59 smithi149 ceph-mon[49797]: Deploying daemon crash.smithi069 on smithi069 2024-04-22T23:17:00.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 174714 11.22 MB/sec execute 230 sec latency 48.393 ms 2024-04-22T23:17:01.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 175036 11.20 MB/sec execute 231 sec latency 45.171 ms 2024-04-22T23:17:01.344 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:01 smithi069 ceph-mon[63133]: pgmap v40: 97 pgs: 97 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.5 MiB/s wr, 362 op/s 2024-04-22T23:17:01.345 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:17:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:01 smithi149 ceph-mon[49797]: pgmap v40: 97 pgs: 97 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.5 MiB/s wr, 362 op/s 2024-04-22T23:17:01.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:17:02.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 175306 11.17 MB/sec execute 232 sec latency 52.864 ms 2024-04-22T23:17:03.131 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:03 smithi069 ceph-mon[63133]: pgmap v41: 97 pgs: 97 active+clean; 323 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.1 MiB/s wr, 508 op/s 2024-04-22T23:17:03.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 175617 11.16 MB/sec execute 233 sec latency 73.862 ms 2024-04-22T23:17:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:03 smithi149 ceph-mon[49797]: pgmap v41: 97 pgs: 97 active+clean; 323 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.1 MiB/s wr, 508 op/s 2024-04-22T23:17:04.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 175980 11.15 MB/sec execute 234 sec latency 54.483 ms 2024-04-22T23:17:05.186 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:04 smithi069 ceph-mon[63133]: pgmap v42: 97 pgs: 97 active+clean; 324 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.8 MiB/s wr, 364 op/s 2024-04-22T23:17:05.186 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:04 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:05.187 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:04 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:05.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:04 smithi149 ceph-mon[49797]: pgmap v42: 97 pgs: 97 active+clean; 324 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.8 MiB/s wr, 364 op/s 2024-04-22T23:17:05.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:04 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:05.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:04 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:05.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 176342 11.13 MB/sec execute 235 sec latency 61.047 ms 2024-04-22T23:17:06.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 176723 11.14 MB/sec execute 236 sec latency 46.498 ms 2024-04-22T23:17:06.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:06 smithi069 ceph-mon[63133]: Upgrade: Updating crash.smithi149 (2/2) 2024-04-22T23:17:06.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:06 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:06.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:06 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:17:06.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:06 smithi069 ceph-mon[63133]: pgmap v43: 97 pgs: 97 active+clean; 324 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.8 MiB/s wr, 364 op/s 2024-04-22T23:17:06.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:06 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:17:06.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:06 smithi069 ceph-mon[63133]: Deploying daemon crash.smithi149 on smithi149 2024-04-22T23:17:06.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:06 smithi149 ceph-mon[49797]: Upgrade: Updating crash.smithi149 (2/2) 2024-04-22T23:17:06.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:06 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:06.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:06 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi149", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-22T23:17:06.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:06 smithi149 ceph-mon[49797]: pgmap v43: 97 pgs: 97 active+clean; 324 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.8 MiB/s wr, 364 op/s 2024-04-22T23:17:06.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:06 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:17:06.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:06 smithi149 ceph-mon[49797]: Deploying daemon crash.smithi149 on smithi149 2024-04-22T23:17:07.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 177075 11.16 MB/sec execute 237 sec latency 36.692 ms 2024-04-22T23:17:08.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 177414 11.16 MB/sec execute 238 sec latency 52.834 ms 2024-04-22T23:17:08.739 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:08 smithi069 ceph-mon[63133]: pgmap v44: 97 pgs: 97 active+clean; 326 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.9 MiB/s wr, 571 op/s 2024-04-22T23:17:09.009 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:08 smithi149 ceph-mon[49797]: pgmap v44: 97 pgs: 97 active+clean; 326 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.9 MiB/s wr, 571 op/s 2024-04-22T23:17:09.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 177846 11.17 MB/sec execute 239 sec latency 38.963 ms 2024-04-22T23:17:10.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 178279 11.15 MB/sec execute 240 sec latency 34.491 ms 2024-04-22T23:17:11.015 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:10 smithi069 ceph-mon[63133]: pgmap v45: 97 pgs: 97 active+clean; 326 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 395 op/s 2024-04-22T23:17:11.016 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:10 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:11.016 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:10 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:11.016 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:10 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:17:11.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:10 smithi149 ceph-mon[49797]: pgmap v45: 97 pgs: 97 active+clean; 326 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 395 op/s 2024-04-22T23:17:11.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:10 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:11.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:10 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:11.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:10 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:17:11.206 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:17:11.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 178690 11.14 MB/sec execute 241 sec latency 48.051 ms 2024-04-22T23:17:11.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:11 smithi149 ceph-mon[49797]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:12.054 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:11 smithi069 ceph-mon[63133]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:12.217 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:17:12.218 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (3m) 77s ago 12m 23.7M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:17:12.218 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 starting - - - - 2024-04-22T23:17:12.218 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 starting - - - - 2024-04-22T23:17:12.218 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (3m) 77s ago 12m 43.5M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:17:12.218 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (6m) 77s ago 6m 38.3M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:17:12.218 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (6m) 60s ago 6m 14.4M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:17:12.218 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (2m) 77s ago 13m 477M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:17:12.218 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (118s) 60s ago 11m 449M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:17:12.218 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (97s) 77s ago 13m 31.3M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:17:12.218 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (67s) 60s ago 11m 21.8M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:17:12.218 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (6m) 77s ago 6m 250M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:17:12.218 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (6m) 60s ago 6m 86.4M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:17:12.219 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (3m) 77s ago 12m 9097k - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:17:12.219 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (3m) 60s ago 11m 16.6M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:17:12.219 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (11m) 77s ago 11m 506M 1936M 17.2.0 e1d6a67b021e 777db7a3d809 2024-04-22T23:17:12.219 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (10m) 77s ago 10m 445M 1936M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:17:12.219 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (10m) 77s ago 10m 333M 1936M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:17:12.219 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (9m) 77s ago 9m 467M 1936M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:17:12.219 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (9m) 60s ago 9m 503M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:17:12.219 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (8m) 60s ago 8m 324M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:17:12.219 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (8m) 60s ago 8m 509M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:17:12.219 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (8m) 60s ago 8m 393M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:17:12.219 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (2m) 77s ago 11m 61.8M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:17:12.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 179050 11.12 MB/sec execute 242 sec latency 46.993 ms 2024-04-22T23:17:12.874 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:17:12.874 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:17:12.874 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:17:12.874 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:17:12.874 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:17:12.874 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:17:12.874 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:17:12.874 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:17:12.875 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T23:17:12.875 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:17:12.875 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:17:12.875 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:17:12.875 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:17:12.875 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:17:12.875 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-22T23:17:12.875 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 4 2024-04-22T23:17:12.875 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:17:12.875 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:17:12.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:12 smithi149 ceph-mon[49797]: pgmap v46: 97 pgs: 97 active+clean; 325 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.4 MiB/s wr, 642 op/s 2024-04-22T23:17:12.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:12 smithi149 ceph-mon[49797]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:12.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:12 smithi149 ceph-mon[49797]: from='client.44117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:13.008 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:12 smithi069 ceph-mon[63133]: pgmap v46: 97 pgs: 97 active+clean; 325 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.4 MiB/s wr, 642 op/s 2024-04-22T23:17:13.009 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:12 smithi069 ceph-mon[63133]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:13.009 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:12 smithi069 ceph-mon[63133]: from='client.44117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:13.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 179534 11.13 MB/sec execute 243 sec latency 61.564 ms 2024-04-22T23:17:13.392 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:17:13.392 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:17:13.392 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:17:13.392 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:17:13.392 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:17:13.392 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:17:13.392 INFO:teuthology.orchestra.run.smithi069.stdout: "mon" 2024-04-22T23:17:13.392 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:17:13.393 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "4/23 daemons upgraded", 2024-04-22T23:17:13.393 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading crash daemons", 2024-04-22T23:17:13.393 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:17:13.393 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:17:14.034 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:13 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/548356266' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:17:14.109 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:17:14.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:13 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/548356266' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:17:14.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 180074 11.14 MB/sec execute 244 sec latency 31.909 ms 2024-04-22T23:17:15.044 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:15 smithi069 ceph-mon[63133]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:15.045 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:15 smithi069 ceph-mon[63133]: pgmap v47: 97 pgs: 97 active+clean; 325 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 496 op/s 2024-04-22T23:17:15.045 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:15 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:15.045 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:15 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:15.045 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:15 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/1811949566' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:17:15.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 180533 11.17 MB/sec execute 245 sec latency 38.109 ms 2024-04-22T23:17:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:15 smithi149 ceph-mon[49797]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:15 smithi149 ceph-mon[49797]: pgmap v47: 97 pgs: 97 active+clean; 325 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 496 op/s 2024-04-22T23:17:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:15 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:15 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:15 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/1811949566' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:17:16.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 181069 11.20 MB/sec execute 246 sec latency 63.967 ms 2024-04-22T23:17:16.564 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:16.564 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:16.564 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:16 smithi069 ceph-mon[63133]: pgmap v48: 97 pgs: 97 active+clean; 325 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 453 op/s 2024-04-22T23:17:16.564 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:17:16.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:16.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:16.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:16 smithi149 ceph-mon[49797]: pgmap v48: 97 pgs: 97 active+clean; 325 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 453 op/s 2024-04-22T23:17:16.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:17:17.278 INFO:teuthology.orchestra.run.smithi069.stdout: 5 181478 11.20 MB/sec execute 247 sec latency 27.794 ms 2024-04-22T23:17:18.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 181876 11.18 MB/sec execute 248 sec latency 37.155 ms 2024-04-22T23:17:18.832 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:18 smithi069 ceph-mon[63133]: pgmap v49: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.6 MiB/s wr, 733 op/s 2024-04-22T23:17:18.832 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:18 smithi069 ceph-mon[63133]: mgrmap e41: smithi069.lyiiqg(active, since 92s), standbys: smithi149.rxqkqk 2024-04-22T23:17:19.105 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:18 smithi149 ceph-mon[49797]: pgmap v49: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.6 MiB/s wr, 733 op/s 2024-04-22T23:17:19.105 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:18 smithi149 ceph-mon[49797]: mgrmap e41: smithi069.lyiiqg(active, since 92s), standbys: smithi149.rxqkqk 2024-04-22T23:17:19.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 182269 11.17 MB/sec execute 249 sec latency 71.138 ms 2024-04-22T23:17:20.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 182671 11.16 MB/sec execute 250 sec latency 42.203 ms 2024-04-22T23:17:20.359 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:20.360 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:20.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:20.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:21.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 183120 11.16 MB/sec execute 251 sec latency 57.681 ms 2024-04-22T23:17:21.374 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:21 smithi069 ceph-mon[63133]: pgmap v50: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.6 MiB/s wr, 526 op/s 2024-04-22T23:17:21.375 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:21 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:21.375 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:21 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:21 smithi149 ceph-mon[49797]: pgmap v50: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.6 MiB/s wr, 526 op/s 2024-04-22T23:17:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:21 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:21 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:22.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 183653 11.17 MB/sec execute 252 sec latency 30.398 ms 2024-04-22T23:17:22.900 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:22 smithi069 ceph-mon[63133]: pgmap v51: 97 pgs: 97 active+clean; 321 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 790 op/s 2024-04-22T23:17:22.901 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:22 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:22.901 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:22 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:23.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:22 smithi149 ceph-mon[49797]: pgmap v51: 97 pgs: 97 active+clean; 321 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 790 op/s 2024-04-22T23:17:23.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:22 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:23.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:22 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:23.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 184125 11.20 MB/sec execute 253 sec latency 30.475 ms 2024-04-22T23:17:24.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 184666 11.23 MB/sec execute 254 sec latency 42.183 ms 2024-04-22T23:17:24.923 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:24 smithi069 ceph-mon[63133]: pgmap v52: 97 pgs: 97 active+clean; 321 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 543 op/s 2024-04-22T23:17:25.136 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:24 smithi149 ceph-mon[49797]: pgmap v52: 97 pgs: 97 active+clean; 321 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 543 op/s 2024-04-22T23:17:25.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 185053 11.22 MB/sec execute 255 sec latency 42.102 ms 2024-04-22T23:17:26.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 185330 11.20 MB/sec execute 256 sec latency 95.352 ms 2024-04-22T23:17:27.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:26 smithi069 ceph-mon[63133]: pgmap v53: 97 pgs: 97 active+clean; 321 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 543 op/s 2024-04-22T23:17:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:26 smithi149 ceph-mon[49797]: pgmap v53: 97 pgs: 97 active+clean; 321 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 543 op/s 2024-04-22T23:17:27.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 185592 11.16 MB/sec execute 257 sec latency 64.490 ms 2024-04-22T23:17:28.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 185856 11.15 MB/sec execute 258 sec latency 62.832 ms 2024-04-22T23:17:29.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: pgmap v54: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 800 op/s 2024-04-22T23:17:29.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:29.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:29.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:17:29.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:17:29.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:29.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: Upgrade: Setting container_image for all crash 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi069"}]: dispatch 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi069"}]': finished 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi149"}]: dispatch 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi149"}]': finished 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T23:17:29.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:29 smithi069 ceph-mon[63133]: Upgrade: osd.0 is safe to restart 2024-04-22T23:17:29.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 186143 11.14 MB/sec execute 259 sec latency 53.912 ms 2024-04-22T23:17:29.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: pgmap v54: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 800 op/s 2024-04-22T23:17:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:17:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:17:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:17:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:17:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:17:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:17:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: Upgrade: Setting container_image for all crash 2024-04-22T23:17:29.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:29.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi069"}]: dispatch 2024-04-22T23:17:29.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi069"}]': finished 2024-04-22T23:17:29.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi149"}]: dispatch 2024-04-22T23:17:29.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi149"}]': finished 2024-04-22T23:17:29.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T23:17:29.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T23:17:29.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:29 smithi149 ceph-mon[49797]: Upgrade: osd.0 is safe to restart 2024-04-22T23:17:30.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 186881 11.17 MB/sec execute 260 sec latency 75.105 ms 2024-04-22T23:17:31.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:31 smithi069 ceph-mon[63133]: pgmap v55: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 520 op/s 2024-04-22T23:17:31.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:31.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T23:17:31.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:17:31.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:17:31.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 187584 11.22 MB/sec execute 261 sec latency 21.322 ms 2024-04-22T23:17:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:31 smithi149 ceph-mon[49797]: pgmap v55: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 520 op/s 2024-04-22T23:17:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T23:17:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:17:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:17:32.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:32 smithi069 ceph-mon[63133]: Upgrade: Updating osd.0 2024-04-22T23:17:32.124 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:32 smithi069 ceph-mon[63133]: Deploying daemon osd.0 on smithi069 2024-04-22T23:17:32.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 187996 11.23 MB/sec execute 262 sec latency 38.520 ms 2024-04-22T23:17:32.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:32 smithi149 ceph-mon[49797]: Upgrade: Updating osd.0 2024-04-22T23:17:32.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:32 smithi149 ceph-mon[49797]: Deploying daemon osd.0 on smithi069 2024-04-22T23:17:33.074 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:33 smithi069 ceph-mon[63133]: pgmap v56: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 775 op/s 2024-04-22T23:17:33.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 188298 11.22 MB/sec execute 263 sec latency 45.363 ms 2024-04-22T23:17:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:33 smithi149 ceph-mon[49797]: pgmap v56: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 775 op/s 2024-04-22T23:17:34.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 188566 11.20 MB/sec execute 264 sec latency 57.614 ms 2024-04-22T23:17:35.187 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:35 smithi069 ceph-mon[63133]: pgmap v57: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 530 op/s 2024-04-22T23:17:35.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 189141 11.20 MB/sec execute 265 sec latency 37.244 ms 2024-04-22T23:17:35.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:35 smithi149 ceph-mon[49797]: pgmap v57: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 530 op/s 2024-04-22T23:17:36.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 189754 11.21 MB/sec execute 266 sec latency 36.889 ms 2024-04-22T23:17:36.462 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:36 smithi069 systemd[1]: Stopping Ceph osd.0 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:17:36.965 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:36 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0[32326]: 2024-04-22T23:17:36.823+0000 7f2f776c3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:17:36.965 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:36 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0[32326]: 2024-04-22T23:17:36.823+0000 7f2f776c3700 -1 osd.0 73 *** Got signal Terminated *** 2024-04-22T23:17:36.965 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:36 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0[32326]: 2024-04-22T23:17:36.823+0000 7f2f776c3700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:17:37.218 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:37 smithi069 ceph-mon[63133]: pgmap v58: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 530 op/s 2024-04-22T23:17:37.218 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:37 smithi069 ceph-mon[63133]: osd.0 marked itself down and dead 2024-04-22T23:17:37.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 190367 11.22 MB/sec execute 267 sec latency 219.184 ms 2024-04-22T23:17:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:37 smithi149 ceph-mon[49797]: pgmap v58: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 530 op/s 2024-04-22T23:17:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:37 smithi149 ceph-mon[49797]: osd.0 marked itself down and dead 2024-04-22T23:17:37.977 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:37 smithi069 podman[68636]: 2024-04-22 23:17:37.709124546 +0000 UTC m=+1.049800867 container died 777db7a3d809456bc081e85a55cb2604985f260381bc78055f4abc53656344b6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0, vcs-type=git, GIT_CLEAN=True, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, version=8, name=centos-stream, io.openshift.expose-services=, GIT_BRANCH=HEAD, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc.) 2024-04-22T23:17:38.227 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:38 smithi069 ceph-mon[63133]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:17:38.227 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:38 smithi069 ceph-mon[63133]: osdmap e74: 8 total, 7 up, 8 in 2024-04-22T23:17:38.227 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:38 smithi069 ceph-mon[63133]: osdmap e75: 8 total, 7 up, 8 in 2024-04-22T23:17:38.227 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:38 smithi069 podman[68636]: 2024-04-22 23:17:38.06680388 +0000 UTC m=+1.407480193 container cleanup 777db7a3d809456bc081e85a55cb2604985f260381bc78055f4abc53656344b6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, release=754, distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.expose-services=, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-22T23:17:38.227 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:38 smithi069 bash[68636]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0 2024-04-22T23:17:38.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 190502 11.19 MB/sec execute 268 sec latency 1031.878 ms 2024-04-22T23:17:38.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:38 smithi149 ceph-mon[49797]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:17:38.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:38 smithi149 ceph-mon[49797]: osdmap e74: 8 total, 7 up, 8 in 2024-04-22T23:17:38.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:38 smithi149 ceph-mon[49797]: osdmap e75: 8 total, 7 up, 8 in 2024-04-22T23:17:38.477 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:38 smithi069 podman[68653]: 2024-04-22 23:17:38.460994882 +0000 UTC m=+0.754258518 container remove 777db7a3d809456bc081e85a55cb2604985f260381bc78055f4abc53656344b6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public) 2024-04-22T23:17:39.055 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:39 smithi069 podman[68771]: 2024-04-22 23:17:38.948945166 +0000 UTC m=+0.024524658 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:17:39.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 191130 11.24 MB/sec execute 269 sec latency 27.448 ms 2024-04-22T23:17:39.313 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:39 smithi069 podman[68771]: 2024-04-22 23:17:39.062114119 +0000 UTC m=+0.137693612 container create d8f079b4ed8e8fbcfc6ef566b3bc46ff5b6cfb409bb1e06d994cab07cae2b4a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T23:17:39.314 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:39 smithi069 ceph-mon[63133]: pgmap v60: 97 pgs: 13 stale+active+clean, 84 active+clean; 327 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.4 MiB/s wr, 666 op/s 2024-04-22T23:17:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:39 smithi149 ceph-mon[49797]: pgmap v60: 97 pgs: 13 stale+active+clean, 84 active+clean; 327 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.4 MiB/s wr, 666 op/s 2024-04-22T23:17:39.565 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:39 smithi069 podman[68771]: 2024-04-22 23:17:39.534243029 +0000 UTC m=+0.609822520 container init d8f079b4ed8e8fbcfc6ef566b3bc46ff5b6cfb409bb1e06d994cab07cae2b4a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:17:39.565 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:39 smithi069 podman[68771]: 2024-04-22 23:17:39.54396531 +0000 UTC m=+0.619544805 container start d8f079b4ed8e8fbcfc6ef566b3bc46ff5b6cfb409bb1e06d994cab07cae2b4a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-22T23:17:39.820 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:39 smithi069 podman[68771]: 2024-04-22 23:17:39.699395497 +0000 UTC m=+0.774974993 container attach d8f079b4ed8e8fbcfc6ef566b3bc46ff5b6cfb409bb1e06d994cab07cae2b4a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T23:17:40.074 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:40 smithi069 podman[68771]: 2024-04-22 23:17:40.025934833 +0000 UTC m=+1.101514332 container died d8f079b4ed8e8fbcfc6ef566b3bc46ff5b6cfb409bb1e06d994cab07cae2b4a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-22T23:17:40.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 191647 11.26 MB/sec execute 270 sec latency 30.582 ms 2024-04-22T23:17:41.084 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:41 smithi069 ceph-mon[63133]: pgmap v62: 97 pgs: 13 stale+active+clean, 84 active+clean; 327 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 449 op/s 2024-04-22T23:17:41.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 192071 11.26 MB/sec execute 271 sec latency 31.953 ms 2024-04-22T23:17:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:41 smithi149 ceph-mon[49797]: pgmap v62: 97 pgs: 13 stale+active+clean, 84 active+clean; 327 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 449 op/s 2024-04-22T23:17:41.602 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:41 smithi069 podman[68796]: 2024-04-22 23:17:41.395535494 +0000 UTC m=+1.362642372 container remove d8f079b4ed8e8fbcfc6ef566b3bc46ff5b6cfb409bb1e06d994cab07cae2b4a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-22T23:17:41.603 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:41 smithi069 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-22T23:17:41.603 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:41 smithi069 systemd[1]: Stopped Ceph osd.0 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:17:41.603 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:41 smithi069 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.0.service: Consumed 3min 12.847s CPU time. 2024-04-22T23:17:41.855 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:41 smithi069 systemd[1]: Starting Ceph osd.0 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:17:42.114 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:41 smithi069 podman[68890]: 2024-04-22 23:17:41.890394221 +0000 UTC m=+0.021293788 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:17:42.114 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 podman[68890]: 2024-04-22 23:17:42.028890938 +0000 UTC m=+0.159790470 container create 2c72ddbe6c6624d22b86b3eecb30f33e85dda21951c1e0c7acf10c7cca857c0e (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-22T23:17:42.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 192293 11.24 MB/sec execute 272 sec latency 71.346 ms 2024-04-22T23:17:42.379 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:42 smithi069 ceph-mon[63133]: Health check failed: Degraded data redundancy: 234/1737 objects degraded (13.472%), 29 pgs degraded (PG_DEGRADED) 2024-04-22T23:17:42.379 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 podman[68890]: 2024-04-22 23:17:42.323618416 +0000 UTC m=+0.454517949 container init 2c72ddbe6c6624d22b86b3eecb30f33e85dda21951c1e0c7acf10c7cca857c0e (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-22T23:17:42.379 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 podman[68890]: 2024-04-22 23:17:42.333322883 +0000 UTC m=+0.464222421 container start 2c72ddbe6c6624d22b86b3eecb30f33e85dda21951c1e0c7acf10c7cca857c0e (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-22T23:17:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:42 smithi149 ceph-mon[49797]: Health check failed: Degraded data redundancy: 234/1737 objects degraded (13.472%), 29 pgs degraded (PG_DEGRADED) 2024-04-22T23:17:42.630 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 podman[68890]: 2024-04-22 23:17:42.450210422 +0000 UTC m=+0.581109955 container attach 2c72ddbe6c6624d22b86b3eecb30f33e85dda21951c1e0c7acf10c7cca857c0e (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-22T23:17:43.161 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:43 smithi069 ceph-mon[63133]: pgmap v63: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.3 MiB/s wr, 802 op/s; 234/1737 objects degraded (13.472%) 2024-04-22T23:17:43.162 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate[68904]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:17:43.162 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 bash[68890]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:17:43.162 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 bash[68890]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:17:43.162 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 bash[68890]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T23:17:43.162 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 bash[68890]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T23:17:43.162 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 bash[68890]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T23:17:43.162 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 bash[68890]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T23:17:43.162 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 bash[68890]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-22T23:17:43.162 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 bash[68890]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T23:17:43.162 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 bash[68890]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-22T23:17:43.162 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate[68904]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:17:43.163 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate[68904]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T23:17:43.163 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate[68904]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T23:17:43.163 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate[68904]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T23:17:43.163 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate[68904]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T23:17:43.163 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate[68904]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-22T23:17:43.163 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate[68904]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T23:17:43.163 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate[68904]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-22T23:17:43.163 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:42 smithi069 podman[68890]: 2024-04-22 23:17:42.951781188 +0000 UTC m=+1.082680732 container died 2c72ddbe6c6624d22b86b3eecb30f33e85dda21951c1e0c7acf10c7cca857c0e (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-22T23:17:43.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 192492 11.21 MB/sec execute 273 sec latency 72.905 ms 2024-04-22T23:17:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:43 smithi149 ceph-mon[49797]: pgmap v63: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.3 MiB/s wr, 802 op/s; 234/1737 objects degraded (13.472%) 2024-04-22T23:17:43.957 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:43 smithi069 podman[69042]: 2024-04-22 23:17:43.787396289 +0000 UTC m=+0.829003964 container remove 2c72ddbe6c6624d22b86b3eecb30f33e85dda21951c1e0c7acf10c7cca857c0e (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-22T23:17:44.215 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:44 smithi069 podman[69099]: 2024-04-22 23:17:43.923576459 +0000 UTC m=+0.020432694 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:17:44.215 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:44 smithi069 podman[69099]: 2024-04-22 23:17:44.090482444 +0000 UTC m=+0.187338665 container create 881ec8fb453d76bb90c613fab3fde5742aba7a500852923348d77b8fb667c20b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-22T23:17:44.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 192740 11.18 MB/sec execute 274 sec latency 49.018 ms 2024-04-22T23:17:44.478 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:44 smithi069 podman[69099]: 2024-04-22 23:17:44.411083446 +0000 UTC m=+0.507939670 container init 881ec8fb453d76bb90c613fab3fde5742aba7a500852923348d77b8fb667c20b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-04-22T23:17:44.478 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:44 smithi069 podman[69099]: 2024-04-22 23:17:44.421387933 +0000 UTC m=+0.518244159 container start 881ec8fb453d76bb90c613fab3fde5742aba7a500852923348d77b8fb667c20b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-22T23:17:44.478 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:44 smithi069 ceph-osd[69140]: -- 172.21.15.69:0/3662291191 <== mon.0 v2:172.21.15.69:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ca02b35680 con 0x55ca02aff000 2024-04-22T23:17:44.731 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:44 smithi069 bash[69099]: 881ec8fb453d76bb90c613fab3fde5742aba7a500852923348d77b8fb667c20b 2024-04-22T23:17:44.732 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:44 smithi069 systemd[1]: Started Ceph osd.0 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:17:44.737 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:17:45.237 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:45 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0[69136]: 2024-04-22T23:17:45.011+0000 7f685b9bc7c0 -1 Falling back to public interface 2024-04-22T23:17:45.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:45 smithi069 ceph-mon[63133]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.3 MiB/s wr, 802 op/s; 234/1737 objects degraded (13.472%) 2024-04-22T23:17:45.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:45.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:45.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:17:45.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 193024 11.17 MB/sec execute 275 sec latency 119.416 ms 2024-04-22T23:17:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:45 smithi149 ceph-mon[49797]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.3 MiB/s wr, 802 op/s; 234/1737 objects degraded (13.472%) 2024-04-22T23:17:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:17:45.771 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:17:45.771 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (3m) 25s ago 13m 23.7M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:17:45.771 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (42s) 25s ago 13m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:17:45.771 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (36s) 31s ago 12m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:17:45.771 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (3m) 25s ago 12m 53.0M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (7m) 25s ago 7m 37.9M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (7m) 31s ago 7m 14.6M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (3m) 25s ago 14m 498M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (2m) 31s ago 12m 450M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (2m) 25s ago 14m 43.3M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (101s) 31s ago 12m 34.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (6m) 25s ago 6m 250M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (6m) 31s ago 6m 86.5M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (3m) 25s ago 12m 20.7M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (3m) 31s ago 12m 16.6M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 starting - - - 1936M 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (11m) 25s ago 11m 552M 1936M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (10m) 25s ago 10m 422M 1936M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:17:45.772 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (10m) 25s ago 10m 536M 1936M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:17:45.773 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (9m) 31s ago 9m 547M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:17:45.773 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (9m) 31s ago 9m 398M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:17:45.773 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (9m) 31s ago 9m 569M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:17:45.773 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (8m) 31s ago 8m 476M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:17:45.773 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (2m) 25s ago 11m 62.5M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:17:46.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:46 smithi069 ceph-mon[63133]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:46.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:46 smithi069 ceph-mon[63133]: from='client.44137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:46.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:46.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:17:46.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 193267 11.13 MB/sec execute 276 sec latency 54.455 ms 2024-04-22T23:17:46.406 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:17:46.406 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:17:46.406 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:17:46.406 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:17:46.407 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:17:46.407 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:17:46.407 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:17:46.407 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:17:46.407 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-04-22T23:17:46.407 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:17:46.408 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:17:46.408 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:17:46.408 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:17:46.408 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:17:46.408 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-22T23:17:46.408 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 4 2024-04-22T23:17:46.409 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:17:46.409 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:17:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:46 smithi149 ceph-mon[49797]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:46 smithi149 ceph-mon[49797]: from='client.44137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:17:46.943 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:17:46.943 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:17:46.943 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:17:46.943 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:17:46.943 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:17:46.944 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:17:46.944 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:17:46.944 INFO:teuthology.orchestra.run.smithi069.stdout: "mon" 2024-04-22T23:17:46.944 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:17:46.944 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "6/23 daemons upgraded", 2024-04-22T23:17:46.944 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T23:17:46.944 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:17:46.944 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:17:47.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:47 smithi069 ceph-mon[63133]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.1 MiB/s wr, 352 op/s; 234/1737 objects degraded (13.472%) 2024-04-22T23:17:47.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:47 smithi069 ceph-mon[63133]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:47.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:47 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/2317322285' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:17:47.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 193596 11.13 MB/sec execute 277 sec latency 54.869 ms 2024-04-22T23:17:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:47 smithi149 ceph-mon[49797]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.1 MiB/s wr, 352 op/s; 234/1737 objects degraded (13.472%) 2024-04-22T23:17:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:47 smithi149 ceph-mon[49797]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:47 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/2317322285' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:17:47.595 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 234/1737 objects degraded (13.472%), 29 pgs degraded 2024-04-22T23:17:47.596 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-22T23:17:47.596 INFO:teuthology.orchestra.run.smithi069.stdout: osd.0 (root=default,host=smithi069) is down 2024-04-22T23:17:47.596 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 234/1737 objects degraded (13.472%), 29 pgs degraded 2024-04-22T23:17:47.596 INFO:teuthology.orchestra.run.smithi069.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-04-22T23:17:47.596 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-04-22T23:17:47.596 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-04-22T23:17:47.596 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-04-22T23:17:47.596 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-04-22T23:17:47.596 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-04-22T23:17:47.596 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-04-22T23:17:47.596 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-04-22T23:17:47.596 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-04-22T23:17:47.597 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-04-22T23:17:47.597 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-04-22T23:17:47.597 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-04-22T23:17:47.597 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-04-22T23:17:47.597 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-04-22T23:17:47.597 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-04-22T23:17:47.597 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-04-22T23:17:47.597 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-04-22T23:17:47.597 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-04-22T23:17:47.597 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-04-22T23:17:47.597 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-04-22T23:17:47.597 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-04-22T23:17:47.598 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-04-22T23:17:47.598 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-04-22T23:17:47.598 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-04-22T23:17:47.598 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-04-22T23:17:47.598 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-04-22T23:17:47.598 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-04-22T23:17:47.598 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-04-22T23:17:47.599 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-04-22T23:17:48.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:48 smithi069 ceph-mon[63133]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:48.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:48 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/2602863861' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:17:48.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 193965 11.13 MB/sec execute 278 sec latency 40.120 ms 2024-04-22T23:17:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:48 smithi149 ceph-mon[49797]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:17:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:48 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/2602863861' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:17:49.281 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:49 smithi069 ceph-mon[63133]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 320 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 511 op/s; 193/1482 objects degraded (13.023%) 2024-04-22T23:17:49.281 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:49 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 193/1482 objects degraded (13.023%), 29 pgs degraded (PG_DEGRADED) 2024-04-22T23:17:49.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 194278 11.12 MB/sec execute 279 sec latency 60.356 ms 2024-04-22T23:17:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:49 smithi149 ceph-mon[49797]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 320 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 511 op/s; 193/1482 objects degraded (13.023%) 2024-04-22T23:17:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:49 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 193/1482 objects degraded (13.023%), 29 pgs degraded (PG_DEGRADED) 2024-04-22T23:17:50.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 194795 11.15 MB/sec execute 280 sec latency 25.190 ms 2024-04-22T23:17:51.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 195252 11.17 MB/sec execute 281 sec latency 27.020 ms 2024-04-22T23:17:51.318 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:51 smithi069 ceph-mon[63133]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 320 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.4 MiB/s wr, 439 op/s; 193/1482 objects degraded (13.023%) 2024-04-22T23:17:51.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:51 smithi149 ceph-mon[49797]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 320 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.4 MiB/s wr, 439 op/s; 193/1482 objects degraded (13.023%) 2024-04-22T23:17:52.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 195576 11.16 MB/sec execute 282 sec latency 40.493 ms 2024-04-22T23:17:52.841 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:52 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0[69136]: 2024-04-22T23:17:52.676+0000 7f685b9bc7c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-22T23:17:53.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 195849 11.14 MB/sec execute 283 sec latency 45.888 ms 2024-04-22T23:17:53.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:53 smithi069 ceph-mon[63133]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.0 MiB/s wr, 653 op/s; 226/1707 objects degraded (13.240%) 2024-04-22T23:17:53.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:53 smithi149 ceph-mon[49797]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.0 MiB/s wr, 653 op/s; 226/1707 objects degraded (13.240%) 2024-04-22T23:17:54.122 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:53 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0[69136]: 2024-04-22T23:17:53.924+0000 7f685b9bc7c0 -1 osd.0 73 log_to_monitors true 2024-04-22T23:17:54.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 196126 11.11 MB/sec execute 284 sec latency 46.008 ms 2024-04-22T23:17:54.889 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:54 smithi069 ceph-mon[63133]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.0 MiB/s wr, 399 op/s; 226/1707 objects degraded (13.240%) 2024-04-22T23:17:54.890 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:54 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:54.890 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:54 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:54.890 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:54 smithi069 ceph-mon[63133]: from='osd.0 [v2:172.21.15.69:6802/173454418,v1:172.21.15.69:6803/173454418]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T23:17:55.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:54 smithi149 ceph-mon[49797]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.0 MiB/s wr, 399 op/s; 226/1707 objects degraded (13.240%) 2024-04-22T23:17:55.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:54 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:55.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:54 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:55.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:54 smithi149 ceph-mon[49797]: from='osd.0 [v2:172.21.15.69:6802/173454418,v1:172.21.15.69:6803/173454418]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T23:17:55.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 196376 11.09 MB/sec execute 285 sec latency 182.184 ms 2024-04-22T23:17:55.645 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:17:55 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0[69136]: 2024-04-22T23:17:55.402+0000 7f6848e0c700 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:17:55.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:55 smithi069 ceph-mon[63133]: from='osd.0 [v2:172.21.15.69:6802/173454418,v1:172.21.15.69:6803/173454418]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T23:17:55.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:55 smithi069 ceph-mon[63133]: osdmap e76: 8 total, 7 up, 8 in 2024-04-22T23:17:55.900 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:55 smithi069 ceph-mon[63133]: from='osd.0 [v2:172.21.15.69:6802/173454418,v1:172.21.15.69:6803/173454418]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:17:55.900 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:55 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 226/1707 objects degraded (13.240%), 29 pgs degraded (PG_DEGRADED) 2024-04-22T23:17:55.900 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:55 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:55.900 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:55 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:55.900 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:55 smithi069 ceph-mon[63133]: from='osd.0 [v2:172.21.15.69:6802/173454418,v1:172.21.15.69:6803/173454418]' entity='osd.0' 2024-04-22T23:17:56.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:55 smithi149 ceph-mon[49797]: from='osd.0 [v2:172.21.15.69:6802/173454418,v1:172.21.15.69:6803/173454418]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T23:17:56.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:55 smithi149 ceph-mon[49797]: osdmap e76: 8 total, 7 up, 8 in 2024-04-22T23:17:56.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:55 smithi149 ceph-mon[49797]: from='osd.0 [v2:172.21.15.69:6802/173454418,v1:172.21.15.69:6803/173454418]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:17:56.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:55 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 226/1707 objects degraded (13.240%), 29 pgs degraded (PG_DEGRADED) 2024-04-22T23:17:56.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:55 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:56.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:55 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:17:56.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:55 smithi149 ceph-mon[49797]: from='osd.0 [v2:172.21.15.69:6802/173454418,v1:172.21.15.69:6803/173454418]' entity='osd.0' 2024-04-22T23:17:56.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 196844 11.08 MB/sec execute 286 sec latency 145.051 ms 2024-04-22T23:17:57.143 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:56 smithi069 ceph-mon[63133]: pgmap v71: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 479 op/s; 226/1707 objects degraded (13.240%) 2024-04-22T23:17:57.143 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:56 smithi069 ceph-mon[63133]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:17:57.143 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:56 smithi069 ceph-mon[63133]: osd.0 [v2:172.21.15.69:6802/173454418,v1:172.21.15.69:6803/173454418] boot 2024-04-22T23:17:57.143 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:56 smithi069 ceph-mon[63133]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T23:17:57.143 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:17:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:56 smithi149 ceph-mon[49797]: pgmap v71: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 479 op/s; 226/1707 objects degraded (13.240%) 2024-04-22T23:17:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:56 smithi149 ceph-mon[49797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:17:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:56 smithi149 ceph-mon[49797]: osd.0 [v2:172.21.15.69:6802/173454418,v1:172.21.15.69:6803/173454418] boot 2024-04-22T23:17:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:56 smithi149 ceph-mon[49797]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T23:17:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T23:17:57.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 196984 11.06 MB/sec execute 287 sec latency 883.457 ms 2024-04-22T23:17:58.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 196990 11.02 MB/sec execute 288 sec latency 1883.484 ms 2024-04-22T23:17:58.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:58 smithi069 ceph-mon[63133]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T23:17:58.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:58 smithi069 ceph-mon[63133]: pgmap v74: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 2.9 MiB/s wr, 385 op/s; 93/1650 objects degraded (5.636%) 2024-04-22T23:17:58.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:58 smithi149 ceph-mon[49797]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T23:17:58.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:58 smithi149 ceph-mon[49797]: pgmap v74: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 2.9 MiB/s wr, 385 op/s; 93/1650 objects degraded (5.636%) 2024-04-22T23:17:59.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 197371 11.02 MB/sec execute 289 sec latency 2069.776 ms 2024-04-22T23:17:59.414 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:17:59 smithi069 ceph-mon[63133]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T23:17:59.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:17:59 smithi149 ceph-mon[49797]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T23:18:00.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 197877 11.03 MB/sec execute 290 sec latency 281.451 ms 2024-04-22T23:18:00.447 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:00 smithi069 ceph-mon[63133]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T23:18:00.448 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:00 smithi069 ceph-mon[63133]: pgmap v77: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.4 MiB/s wr, 578 op/s; 93/1650 objects degraded (5.636%) 2024-04-22T23:18:00.448 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:00 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 93/1650 objects degraded (5.636%), 8 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:00.448 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:00 smithi069 ceph-mon[63133]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T23:18:00.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:00 smithi149 ceph-mon[49797]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T23:18:00.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:00 smithi149 ceph-mon[49797]: pgmap v77: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.4 MiB/s wr, 578 op/s; 93/1650 objects degraded (5.636%) 2024-04-22T23:18:00.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:00 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 93/1650 objects degraded (5.636%), 8 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:00.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:00 smithi149 ceph-mon[49797]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T23:18:01.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 198246 11.04 MB/sec execute 291 sec latency 1025.098 ms 2024-04-22T23:18:01.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:01.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:18:01.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:01.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:01.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:18:01.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T23:18:01.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:18:01.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:01.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:01.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:18:01.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:01.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:01.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:01.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T23:18:01.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T23:18:01.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:01 smithi069 ceph-mon[63133]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-22T23:18:02.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:02.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T23:18:02.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T23:18:02.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:01 smithi149 ceph-mon[49797]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-22T23:18:02.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 198702 11.06 MB/sec execute 292 sec latency 24.941 ms 2024-04-22T23:18:03.226 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:03 smithi069 ceph-mon[63133]: pgmap v80: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 89 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.2 MiB/s wr, 503 op/s; 149/1626 objects degraded (9.164%); 3.0 MiB/s, 29 objects/s recovering 2024-04-22T23:18:03.227 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:03 smithi069 ceph-mon[63133]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T23:18:03.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 199079 11.06 MB/sec execute 293 sec latency 35.273 ms 2024-04-22T23:18:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:03 smithi149 ceph-mon[49797]: pgmap v80: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 89 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.2 MiB/s wr, 503 op/s; 149/1626 objects degraded (9.164%); 3.0 MiB/s, 29 objects/s recovering 2024-04-22T23:18:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:03 smithi149 ceph-mon[49797]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T23:18:04.240 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:04 smithi069 ceph-mon[63133]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T23:18:04.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 199492 11.05 MB/sec execute 294 sec latency 219.164 ms 2024-04-22T23:18:04.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:04 smithi149 ceph-mon[49797]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T23:18:05.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:05 smithi069 ceph-mon[63133]: pgmap v83: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 89 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.2 MiB/s wr, 503 op/s; 149/1626 objects degraded (9.164%); 3.0 MiB/s, 29 objects/s recovering 2024-04-22T23:18:05.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:05 smithi069 ceph-mon[63133]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T23:18:05.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:05 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 149/1626 objects degraded (9.164%), 7 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:05.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:05 smithi069 ceph-mon[63133]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T23:18:05.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 199829 11.03 MB/sec execute 295 sec latency 481.228 ms 2024-04-22T23:18:05.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:05 smithi149 ceph-mon[49797]: pgmap v83: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 89 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.2 MiB/s wr, 503 op/s; 149/1626 objects degraded (9.164%); 3.0 MiB/s, 29 objects/s recovering 2024-04-22T23:18:05.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:05 smithi149 ceph-mon[49797]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T23:18:05.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:05 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 149/1626 objects degraded (9.164%), 7 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:05.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:05 smithi149 ceph-mon[49797]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T23:18:06.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 200119 11.02 MB/sec execute 296 sec latency 841.620 ms 2024-04-22T23:18:07.027 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:07 smithi069 ceph-mon[63133]: pgmap v86: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.8 MiB/s wr, 300 op/s; 117/1644 objects degraded (7.117%); 0 B/s, 11 objects/s recovering 2024-04-22T23:18:07.028 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:07 smithi069 ceph-mon[63133]: osdmap e87: 8 total, 8 up, 8 in 2024-04-22T23:18:07.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 200445 11.00 MB/sec execute 297 sec latency 303.454 ms 2024-04-22T23:18:07.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:07 smithi149 ceph-mon[49797]: pgmap v86: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.8 MiB/s wr, 300 op/s; 117/1644 objects degraded (7.117%); 0 B/s, 11 objects/s recovering 2024-04-22T23:18:07.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:07 smithi149 ceph-mon[49797]: osdmap e87: 8 total, 8 up, 8 in 2024-04-22T23:18:08.045 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:08 smithi069 ceph-mon[63133]: osdmap e88: 8 total, 8 up, 8 in 2024-04-22T23:18:08.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 200813 10.99 MB/sec execute 298 sec latency 48.510 ms 2024-04-22T23:18:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:08 smithi149 ceph-mon[49797]: osdmap e88: 8 total, 8 up, 8 in 2024-04-22T23:18:09.087 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:09 smithi069 ceph-mon[63133]: pgmap v89: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 6.2 MiB/s wr, 724 op/s; 31/1551 objects degraded (1.999%); 0 B/s, 34 objects/s recovering 2024-04-22T23:18:09.087 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:09 smithi069 ceph-mon[63133]: osdmap e89: 8 total, 8 up, 8 in 2024-04-22T23:18:09.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201167 11.00 MB/sec execute 299 sec latency 51.259 ms 2024-04-22T23:18:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:09 smithi149 ceph-mon[49797]: pgmap v89: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 6.2 MiB/s wr, 724 op/s; 31/1551 objects degraded (1.999%); 0 B/s, 34 objects/s recovering 2024-04-22T23:18:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:09 smithi149 ceph-mon[49797]: osdmap e89: 8 total, 8 up, 8 in 2024-04-22T23:18:10.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:10 smithi069 ceph-mon[63133]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T23:18:10.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:10 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 31/1551 objects degraded (1.999%), 2 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:10.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201576 11.00 MB/sec execute 300 sec latency 32.614 ms 2024-04-22T23:18:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:10 smithi149 ceph-mon[49797]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T23:18:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:10 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 31/1551 objects degraded (1.999%), 2 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:11.099 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:11 smithi069 ceph-mon[63133]: pgmap v92: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 4.5 MiB/s wr, 423 op/s; 31/1551 objects degraded (1.999%); 0 B/s, 22 objects/s recovering 2024-04-22T23:18:11.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201916 11.01 MB/sec execute 301 sec latency 42.056 ms 2024-04-22T23:18:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:11 smithi149 ceph-mon[49797]: pgmap v92: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 4.5 MiB/s wr, 423 op/s; 31/1551 objects degraded (1.999%); 0 B/s, 22 objects/s recovering 2024-04-22T23:18:12.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:12 smithi069 ceph-mon[63133]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/1551 objects degraded (1.999%), 2 pgs degraded) 2024-04-22T23:18:12.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:12 smithi069 ceph-mon[63133]: Cluster is now healthy 2024-04-22T23:18:12.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 202233 11.02 MB/sec execute 302 sec latency 36.396 ms 2024-04-22T23:18:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:12 smithi149 ceph-mon[49797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/1551 objects degraded (1.999%), 2 pgs degraded) 2024-04-22T23:18:12.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:12 smithi149 ceph-mon[49797]: Cluster is now healthy 2024-04-22T23:18:13.122 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:13 smithi069 ceph-mon[63133]: pgmap v93: 97 pgs: 97 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 7.9 MiB/s wr, 742 op/s; 0 B/s, 21 objects/s recovering 2024-04-22T23:18:13.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 202515 11.00 MB/sec execute 303 sec latency 57.952 ms 2024-04-22T23:18:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:13 smithi149 ceph-mon[49797]: pgmap v93: 97 pgs: 97 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 7.9 MiB/s wr, 742 op/s; 0 B/s, 21 objects/s recovering 2024-04-22T23:18:14.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 202740 10.98 MB/sec execute 304 sec latency 60.513 ms 2024-04-22T23:18:15.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:15 smithi069 ceph-mon[63133]: pgmap v94: 97 pgs: 97 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.1 MiB/s wr, 379 op/s; 0 B/s, 4 objects/s recovering 2024-04-22T23:18:15.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 202939 10.96 MB/sec execute 305 sec latency 63.541 ms 2024-04-22T23:18:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:15 smithi149 ceph-mon[49797]: pgmap v94: 97 pgs: 97 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.1 MiB/s wr, 379 op/s; 0 B/s, 4 objects/s recovering 2024-04-22T23:18:16.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 203539 10.96 MB/sec execute 306 sec latency 40.728 ms 2024-04-22T23:18:16.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:16 smithi069 ceph-mon[63133]: pgmap v95: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.1 MiB/s wr, 336 op/s; 0 B/s, 4 objects/s recovering 2024-04-22T23:18:16.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:16.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:18:16.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T23:18:16.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:16 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T23:18:16.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:16 smithi069 ceph-mon[63133]: Upgrade: osd.1 is safe to restart 2024-04-22T23:18:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:16 smithi149 ceph-mon[49797]: pgmap v95: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.1 MiB/s wr, 336 op/s; 0 B/s, 4 objects/s recovering 2024-04-22T23:18:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:18:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T23:18:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:16 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T23:18:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:16 smithi149 ceph-mon[49797]: Upgrade: osd.1 is safe to restart 2024-04-22T23:18:17.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204189 10.96 MB/sec execute 307 sec latency 36.153 ms 2024-04-22T23:18:18.206 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:18:18.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204930 11.00 MB/sec execute 308 sec latency 34.031 ms 2024-04-22T23:18:18.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:18 smithi069 ceph-mon[63133]: pgmap v96: 97 pgs: 97 active+clean; 332 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 532 op/s; 0 B/s, 3 objects/s recovering 2024-04-22T23:18:18.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:18 smithi069 ceph-mon[63133]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:19.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:18 smithi149 ceph-mon[49797]: pgmap v96: 97 pgs: 97 active+clean; 332 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 532 op/s; 0 B/s, 3 objects/s recovering 2024-04-22T23:18:19.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:18 smithi149 ceph-mon[49797]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:19.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 205500 11.04 MB/sec execute 309 sec latency 41.304 ms 2024-04-22T23:18:19.333 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:18:19.333 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (4m) 25s ago 13m 23.7M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:18:19.333 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (76s) 25s ago 13m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:18:19.333 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (69s) 65s ago 13m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:18:19.333 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (4m) 25s ago 13m 53.2M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:18:19.333 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (7m) 25s ago 7m 38.2M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:18:19.333 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (7m) 65s ago 7m 14.6M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:18:19.333 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (4m) 25s ago 14m 500M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:18:19.333 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (3m) 65s ago 12m 450M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (2m) 25s ago 15m 45.6M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (2m) 65s ago 12m 34.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (7m) 25s ago 7m 253M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (7m) 65s ago 7m 86.5M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (4m) 25s ago 13m 18.8M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (4m) 65s ago 12m 16.6M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (35s) 25s ago 12m 11.5M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (11m) 25s ago 11m 587M 1936M 17.2.0 e1d6a67b021e cfea41a7d7b9 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (11m) 25s ago 11m 481M 1936M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (10m) 25s ago 10m 584M 1936M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (10m) 65s ago 10m 547M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (10m) 65s ago 9m 398M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (9m) 65s ago 9m 569M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:18:19.334 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (9m) 65s ago 9m 476M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:18:19.335 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (3m) 25s ago 12m 67.1M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:18:20.022 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:18:20.022 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:18:20.022 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:18:20.022 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:18:20.022 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:18:20.022 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:18:20.022 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:18:20.023 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:18:20.023 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-22T23:18:20.023 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 1 2024-04-22T23:18:20.023 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:18:20.023 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:18:20.023 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:18:20.023 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:18:20.023 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:18:20.023 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-22T23:18:20.023 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 5 2024-04-22T23:18:20.023 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:18:20.023 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:18:20.237 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:20 smithi069 ceph-mon[63133]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:20.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:20 smithi069 ceph-mon[63133]: Upgrade: Updating osd.1 2024-04-22T23:18:20.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:20.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T23:18:20.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:18:20.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:20 smithi069 ceph-mon[63133]: Deploying daemon osd.1 on smithi069 2024-04-22T23:18:20.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:20 smithi069 ceph-mon[63133]: from='client.44161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:20.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 205847 11.05 MB/sec execute 310 sec latency 74.997 ms 2024-04-22T23:18:20.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:20 smithi149 ceph-mon[49797]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:20.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:20 smithi149 ceph-mon[49797]: Upgrade: Updating osd.1 2024-04-22T23:18:20.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:20.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T23:18:20.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:18:20.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:20 smithi149 ceph-mon[49797]: Deploying daemon osd.1 on smithi069 2024-04-22T23:18:20.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:20 smithi149 ceph-mon[49797]: from='client.44161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:20.557 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:18:20.558 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:18:20.558 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:18:20.558 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:18:20.558 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:18:20.558 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:18:20.558 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:18:20.558 INFO:teuthology.orchestra.run.smithi069.stdout: "mon" 2024-04-22T23:18:20.558 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:18:20.558 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "7/23 daemons upgraded", 2024-04-22T23:18:20.558 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T23:18:20.558 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:18:20.558 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:18:21.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:21 smithi069 ceph-mon[63133]: pgmap v97: 97 pgs: 97 active+clean; 332 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.9 MiB/s wr, 483 op/s; 0 B/s, 3 objects/s recovering 2024-04-22T23:18:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:21 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/3909163119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:21 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:21.285 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:18:21.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 206184 11.04 MB/sec execute 311 sec latency 33.963 ms 2024-04-22T23:18:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:21 smithi149 ceph-mon[49797]: pgmap v97: 97 pgs: 97 active+clean; 332 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.9 MiB/s wr, 483 op/s; 0 B/s, 3 objects/s recovering 2024-04-22T23:18:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:21 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/3909163119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:21 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:22.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:22 smithi069 ceph-mon[63133]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:22.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:22 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/1841839084' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:18:22.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 206463 11.02 MB/sec execute 312 sec latency 43.044 ms 2024-04-22T23:18:22.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:22 smithi149 ceph-mon[49797]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:22.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:22 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/1841839084' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:18:23.046 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:23 smithi069 ceph-mon[63133]: pgmap v98: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.7 MiB/s wr, 726 op/s; 0 B/s, 2 objects/s recovering 2024-04-22T23:18:23.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 206819 11.00 MB/sec execute 313 sec latency 54.226 ms 2024-04-22T23:18:23.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:23 smithi149 ceph-mon[49797]: pgmap v98: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.7 MiB/s wr, 726 op/s; 0 B/s, 2 objects/s recovering 2024-04-22T23:18:24.057 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:24 smithi069 systemd[1]: Stopping Ceph osd.1 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:18:24.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 207264 11.00 MB/sec execute 314 sec latency 82.399 ms 2024-04-22T23:18:24.564 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:24 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[36372]: 2024-04-22T23:18:24.478+0000 7f1d717c9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:18:24.565 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:24 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[36372]: 2024-04-22T23:18:24.478+0000 7f1d717c9700 -1 osd.1 90 *** Got signal Terminated *** 2024-04-22T23:18:24.565 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:24 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[36372]: 2024-04-22T23:18:24.478+0000 7f1d717c9700 -1 osd.1 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:18:25.067 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:25 smithi069 ceph-mon[63133]: pgmap v99: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 514 op/s 2024-04-22T23:18:25.067 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:25 smithi069 ceph-mon[63133]: osd.1 marked itself down and dead 2024-04-22T23:18:25.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 207628 10.98 MB/sec execute 315 sec latency 232.320 ms 2024-04-22T23:18:25.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:25 smithi149 ceph-mon[49797]: pgmap v99: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 514 op/s 2024-04-22T23:18:25.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:25 smithi149 ceph-mon[49797]: osd.1 marked itself down and dead 2024-04-22T23:18:25.983 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:25 smithi069 podman[71376]: 2024-04-22 23:18:25.681281569 +0000 UTC m=+1.419020770 container died cfea41a7d7b9757507674c3d11c91049e021283234e4974a5861c3149cd73de6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, name=centos-stream, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , RELEASE=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T23:18:26.233 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:26 smithi069 ceph-mon[63133]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:18:26.233 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:26 smithi069 ceph-mon[63133]: osdmap e91: 8 total, 7 up, 8 in 2024-04-22T23:18:26.234 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:26 smithi069 podman[71376]: 2024-04-22 23:18:26.098345322 +0000 UTC m=+1.836084527 container cleanup cfea41a7d7b9757507674c3d11c91049e021283234e4974a5861c3149cd73de6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.19.8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, vcs-type=git, ceph=True, GIT_BRANCH=HEAD, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-22T23:18:26.234 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:26 smithi069 bash[71376]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1 2024-04-22T23:18:26.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 207696 10.96 MB/sec execute 316 sec latency 1062.769 ms 2024-04-22T23:18:26.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:26 smithi149 ceph-mon[49797]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:18:26.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:26 smithi149 ceph-mon[49797]: osdmap e91: 8 total, 7 up, 8 in 2024-04-22T23:18:26.493 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:26 smithi069 podman[71395]: 2024-04-22 23:18:26.467630453 +0000 UTC m=+0.782661947 container remove cfea41a7d7b9757507674c3d11c91049e021283234e4974a5861c3149cd73de6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, ceph=True, architecture=x86_64, io.buildah.version=1.19.8, release=754, distribution-scope=public, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-22T23:18:26.999 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:26 smithi069 podman[71486]: 2024-04-22 23:18:26.885440604 +0000 UTC m=+0.023032988 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:18:27.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:27 smithi069 ceph-mon[63133]: pgmap v101: 97 pgs: 14 stale+active+clean, 83 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 621 op/s 2024-04-22T23:18:27.251 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:27 smithi069 ceph-mon[63133]: osdmap e92: 8 total, 7 up, 8 in 2024-04-22T23:18:27.251 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:27 smithi069 podman[71486]: 2024-04-22 23:18:27.038119904 +0000 UTC m=+0.175712294 container create fa310b40dd855d67a28a212d3c838f20e92fe9b9c33a60a444335c379bede094 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240416, ceph=True, org.label-schema.schema-version=1.0) 2024-04-22T23:18:27.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 208110 10.96 MB/sec execute 317 sec latency 43.239 ms 2024-04-22T23:18:27.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:27 smithi149 ceph-mon[49797]: pgmap v101: 97 pgs: 14 stale+active+clean, 83 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 621 op/s 2024-04-22T23:18:27.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:27 smithi149 ceph-mon[49797]: osdmap e92: 8 total, 7 up, 8 in 2024-04-22T23:18:27.504 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:27 smithi069 podman[71486]: 2024-04-22 23:18:27.434199725 +0000 UTC m=+0.571792108 container init fa310b40dd855d67a28a212d3c838f20e92fe9b9c33a60a444335c379bede094 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-22T23:18:27.504 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:27 smithi069 podman[71486]: 2024-04-22 23:18:27.44540467 +0000 UTC m=+0.582997058 container start fa310b40dd855d67a28a212d3c838f20e92fe9b9c33a60a444335c379bede094 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-22T23:18:27.774 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:27 smithi069 podman[71486]: 2024-04-22 23:18:27.707642564 +0000 UTC m=+0.845234949 container attach fa310b40dd855d67a28a212d3c838f20e92fe9b9c33a60a444335c379bede094 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-22T23:18:28.027 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:27 smithi069 podman[71486]: 2024-04-22 23:18:27.893435875 +0000 UTC m=+1.031028273 container died fa310b40dd855d67a28a212d3c838f20e92fe9b9c33a60a444335c379bede094 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T23:18:28.284 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:28 smithi069 ceph-mon[63133]: Health check failed: Degraded data redundancy: 148/1509 objects degraded (9.808%), 22 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:28.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 208563 10.97 MB/sec execute 318 sec latency 46.867 ms 2024-04-22T23:18:28.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:28 smithi149 ceph-mon[49797]: Health check failed: Degraded data redundancy: 148/1509 objects degraded (9.808%), 22 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:29.046 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:28 smithi069 podman[71511]: 2024-04-22 23:18:28.966410159 +0000 UTC m=+1.067568761 container remove fa310b40dd855d67a28a212d3c838f20e92fe9b9c33a60a444335c379bede094 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-22T23:18:29.046 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:28 smithi069 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-22T23:18:29.046 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:28 smithi069 systemd[1]: Stopped Ceph osd.1 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:18:29.046 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:28 smithi069 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.1.service: Consumed 3min 42.780s CPU time. 2024-04-22T23:18:29.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 209002 11.00 MB/sec execute 319 sec latency 72.721 ms 2024-04-22T23:18:29.296 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:29 smithi069 ceph-mon[63133]: pgmap v103: 97 pgs: 22 active+undersized+degraded, 1 stale+active+clean, 14 active+undersized, 60 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 756 op/s; 148/1509 objects degraded (9.808%); 0 B/s, 0 objects/s recovering 2024-04-22T23:18:29.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:29 smithi149 ceph-mon[49797]: pgmap v103: 97 pgs: 22 active+undersized+degraded, 1 stale+active+clean, 14 active+undersized, 60 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 756 op/s; 148/1509 objects degraded (9.808%); 0 B/s, 0 objects/s recovering 2024-04-22T23:18:29.547 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:29 smithi069 systemd[1]: Starting Ceph osd.1 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:18:29.803 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:29 smithi069 podman[71602]: 2024-04-22 23:18:29.563739999 +0000 UTC m=+0.028900439 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:18:29.803 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:29 smithi069 podman[71602]: 2024-04-22 23:18:29.699383784 +0000 UTC m=+0.164544211 container create 9fe54d39524a58ac544a68f12238b5114dd5af8215814286d1f10a5d1408108d (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T23:18:30.054 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:29 smithi069 podman[71602]: 2024-04-22 23:18:29.936121228 +0000 UTC m=+0.401281663 container init 9fe54d39524a58ac544a68f12238b5114dd5af8215814286d1f10a5d1408108d (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD) 2024-04-22T23:18:30.054 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:29 smithi069 podman[71602]: 2024-04-22 23:18:29.945773619 +0000 UTC m=+0.410934048 container start 9fe54d39524a58ac544a68f12238b5114dd5af8215814286d1f10a5d1408108d (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-22T23:18:30.054 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 podman[71602]: 2024-04-22 23:18:30.044419769 +0000 UTC m=+0.509580190 container attach 9fe54d39524a58ac544a68f12238b5114dd5af8215814286d1f10a5d1408108d (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:18:30.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 209429 11.00 MB/sec execute 320 sec latency 34.408 ms 2024-04-22T23:18:30.812 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate[71616]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:18:30.812 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate[71616]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:18:30.812 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 bash[71602]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:18:30.812 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 bash[71602]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:18:30.812 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 bash[71602]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T23:18:30.812 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 bash[71602]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 bash[71602]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 bash[71602]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 bash[71602]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 bash[71602]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 bash[71602]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate[71616]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate[71616]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate[71616]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate[71616]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate[71616]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate[71616]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate[71616]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-22T23:18:30.813 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:30 smithi069 podman[71602]: 2024-04-22 23:18:30.654010471 +0000 UTC m=+1.119170895 container died 9fe54d39524a58ac544a68f12238b5114dd5af8215814286d1f10a5d1408108d (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-22T23:18:31.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 209830 10.99 MB/sec execute 321 sec latency 45.233 ms 2024-04-22T23:18:31.316 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:31 smithi069 ceph-mon[63133]: pgmap v104: 97 pgs: 22 active+undersized+degraded, 1 stale+active+clean, 14 active+undersized, 60 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.6 MiB/s wr, 311 op/s; 148/1509 objects degraded (9.808%); 0 B/s, 0 objects/s recovering 2024-04-22T23:18:31.316 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:31.316 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:18:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:31 smithi149 ceph-mon[49797]: pgmap v104: 97 pgs: 22 active+undersized+degraded, 1 stale+active+clean, 14 active+undersized, 60 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.6 MiB/s wr, 311 op/s; 148/1509 objects degraded (9.808%); 0 B/s, 0 objects/s recovering 2024-04-22T23:18:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:31.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:18:31.827 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:31 smithi069 podman[71754]: 2024-04-22 23:18:31.571125492 +0000 UTC m=+0.908034146 container remove 9fe54d39524a58ac544a68f12238b5114dd5af8215814286d1f10a5d1408108d (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2) 2024-04-22T23:18:32.080 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:31 smithi069 podman[71817]: 2024-04-22 23:18:31.731654742 +0000 UTC m=+0.020710181 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:18:32.081 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:31 smithi069 podman[71817]: 2024-04-22 23:18:31.877961481 +0000 UTC m=+0.167016957 container create 93d880f30ae408c6286afa29cc8d99d13e2da37859f18af9e13f3d2285da4a99 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, GIT_CLEAN=True) 2024-04-22T23:18:32.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 210020 10.97 MB/sec execute 322 sec latency 57.233 ms 2024-04-22T23:18:32.332 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:32 smithi069 podman[71817]: 2024-04-22 23:18:32.274390416 +0000 UTC m=+0.563445845 container init 93d880f30ae408c6286afa29cc8d99d13e2da37859f18af9e13f3d2285da4a99 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-22T23:18:32.333 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:32 smithi069 podman[71817]: 2024-04-22 23:18:32.283972584 +0000 UTC m=+0.573028019 container start 93d880f30ae408c6286afa29cc8d99d13e2da37859f18af9e13f3d2285da4a99 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-22T23:18:32.333 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:32 smithi069 ceph-osd[71836]: -- 172.21.15.69:0/377870577 <== mon.0 v2:172.21.15.69:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56229fd9b680 con 0x56229fd65000 2024-04-22T23:18:32.586 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:32 smithi069 bash[71817]: 93d880f30ae408c6286afa29cc8d99d13e2da37859f18af9e13f3d2285da4a99 2024-04-22T23:18:32.586 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:32 smithi069 systemd[1]: Started Ceph osd.1 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:18:33.105 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:32 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[71832]: 2024-04-22T23:18:32.868+0000 7ffa4309c7c0 -1 Falling back to public interface 2024-04-22T23:18:33.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 210224 10.95 MB/sec execute 323 sec latency 111.865 ms 2024-04-22T23:18:33.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:33 smithi069 ceph-mon[63133]: pgmap v105: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.9 MiB/s wr, 665 op/s; 194/1737 objects degraded (11.169%); 0 B/s, 0 objects/s recovering 2024-04-22T23:18:33.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:33 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:33.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:33 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:33.372 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:33 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:18:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:33 smithi149 ceph-mon[49797]: pgmap v105: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.9 MiB/s wr, 665 op/s; 194/1737 objects degraded (11.169%); 0 B/s, 0 objects/s recovering 2024-04-22T23:18:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:33 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:33 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:33 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:18:34.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 210447 10.93 MB/sec execute 324 sec latency 93.952 ms 2024-04-22T23:18:35.210 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:35 smithi069 ceph-mon[63133]: pgmap v106: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 592 op/s; 194/1737 objects degraded (11.169%); 0 B/s, 0 objects/s recovering 2024-04-22T23:18:35.211 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:35 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 194/1737 objects degraded (11.169%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:35.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 210708 10.92 MB/sec execute 325 sec latency 65.243 ms 2024-04-22T23:18:35.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:35 smithi149 ceph-mon[49797]: pgmap v106: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 592 op/s; 194/1737 objects degraded (11.169%); 0 B/s, 0 objects/s recovering 2024-04-22T23:18:35.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:35 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 194/1737 objects degraded (11.169%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:36.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 211050 10.90 MB/sec execute 326 sec latency 93.441 ms 2024-04-22T23:18:37.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:37 smithi069 ceph-mon[63133]: pgmap v107: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.4 MiB/s wr, 528 op/s; 196/1704 objects degraded (11.502%); 0 B/s, 0 objects/s recovering 2024-04-22T23:18:37.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 211456 10.90 MB/sec execute 327 sec latency 36.890 ms 2024-04-22T23:18:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:37 smithi149 ceph-mon[49797]: pgmap v107: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.4 MiB/s wr, 528 op/s; 196/1704 objects degraded (11.502%); 0 B/s, 0 objects/s recovering 2024-04-22T23:18:38.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 211921 10.91 MB/sec execute 328 sec latency 43.721 ms 2024-04-22T23:18:39.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 212309 10.91 MB/sec execute 329 sec latency 46.433 ms 2024-04-22T23:18:39.292 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:39 smithi069 ceph-mon[63133]: pgmap v108: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 324 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 617 op/s; 167/1518 objects degraded (11.001%); 0 B/s, 0 objects/s recovering 2024-04-22T23:18:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:39 smithi149 ceph-mon[49797]: pgmap v108: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 324 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 617 op/s; 167/1518 objects degraded (11.001%); 0 B/s, 0 objects/s recovering 2024-04-22T23:18:40.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 212626 10.92 MB/sec execute 330 sec latency 38.610 ms 2024-04-22T23:18:40.307 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:40 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 167/1518 objects degraded (11.001%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:40.308 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:40 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[71832]: 2024-04-22T23:18:40.157+0000 7ffa4309c7c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-22T23:18:40.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:40 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 167/1518 objects degraded (11.001%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:41.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 212935 10.92 MB/sec execute 331 sec latency 44.605 ms 2024-04-22T23:18:41.335 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:41 smithi069 ceph-mon[63133]: pgmap v109: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 324 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 407 op/s; 167/1518 objects degraded (11.001%) 2024-04-22T23:18:41.336 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:41 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:41.336 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:41 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:41 smithi149 ceph-mon[49797]: pgmap v109: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 324 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 407 op/s; 167/1518 objects degraded (11.001%) 2024-04-22T23:18:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:41 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:41 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:41.587 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:41 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[71832]: 2024-04-22T23:18:41.543+0000 7ffa4309c7c0 -1 osd.1 90 log_to_monitors true 2024-04-22T23:18:42.169 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:42 smithi069 ceph-mon[63133]: from='osd.1 [v2:172.21.15.69:6810/2600664864,v1:172.21.15.69:6811/2600664864]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T23:18:42.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 213153 10.90 MB/sec execute 332 sec latency 123.672 ms 2024-04-22T23:18:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:42 smithi149 ceph-mon[49797]: from='osd.1 [v2:172.21.15.69:6810/2600664864,v1:172.21.15.69:6811/2600664864]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T23:18:42.892 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:18:42 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[71832]: 2024-04-22T23:18:42.666+0000 7ffa304ec700 -1 osd.1 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:18:43.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 213396 10.89 MB/sec execute 333 sec latency 138.245 ms 2024-04-22T23:18:43.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:43 smithi069 ceph-mon[63133]: pgmap v110: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.6 MiB/s wr, 620 op/s; 184/1689 objects degraded (10.894%) 2024-04-22T23:18:43.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:43 smithi069 ceph-mon[63133]: from='osd.1 [v2:172.21.15.69:6810/2600664864,v1:172.21.15.69:6811/2600664864]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T23:18:43.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:43 smithi069 ceph-mon[63133]: osdmap e93: 8 total, 7 up, 8 in 2024-04-22T23:18:43.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:43 smithi069 ceph-mon[63133]: from='osd.1 [v2:172.21.15.69:6810/2600664864,v1:172.21.15.69:6811/2600664864]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:18:43.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:43.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:43.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:43 smithi069 ceph-mon[63133]: from='osd.1 [v2:172.21.15.69:6810/2600664864,v1:172.21.15.69:6811/2600664864]' entity='osd.1' 2024-04-22T23:18:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:43 smithi149 ceph-mon[49797]: pgmap v110: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.6 MiB/s wr, 620 op/s; 184/1689 objects degraded (10.894%) 2024-04-22T23:18:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:43 smithi149 ceph-mon[49797]: from='osd.1 [v2:172.21.15.69:6810/2600664864,v1:172.21.15.69:6811/2600664864]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T23:18:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:43 smithi149 ceph-mon[49797]: osdmap e93: 8 total, 7 up, 8 in 2024-04-22T23:18:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:43 smithi149 ceph-mon[49797]: from='osd.1 [v2:172.21.15.69:6810/2600664864,v1:172.21.15.69:6811/2600664864]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:18:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:43 smithi149 ceph-mon[49797]: from='osd.1 [v2:172.21.15.69:6810/2600664864,v1:172.21.15.69:6811/2600664864]' entity='osd.1' 2024-04-22T23:18:44.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 213503 10.86 MB/sec execute 334 sec latency 658.842 ms 2024-04-22T23:18:44.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:44 smithi069 ceph-mon[63133]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:18:44.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:44 smithi069 ceph-mon[63133]: osd.1 [v2:172.21.15.69:6810/2600664864,v1:172.21.15.69:6811/2600664864] boot 2024-04-22T23:18:44.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:44 smithi069 ceph-mon[63133]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T23:18:44.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:44 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:18:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:44 smithi149 ceph-mon[49797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:18:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:44 smithi149 ceph-mon[49797]: osd.1 [v2:172.21.15.69:6810/2600664864,v1:172.21.15.69:6811/2600664864] boot 2024-04-22T23:18:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:44 smithi149 ceph-mon[49797]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T23:18:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:44 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T23:18:45.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 213503 10.83 MB/sec execute 335 sec latency 1658.866 ms 2024-04-22T23:18:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:45 smithi149 ceph-mon[49797]: pgmap v113: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 557 op/s; 184/1689 objects degraded (10.894%) 2024-04-22T23:18:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:45 smithi149 ceph-mon[49797]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T23:18:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:45 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 184/1689 objects degraded (10.894%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:45.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:45 smithi069 ceph-mon[63133]: pgmap v113: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 557 op/s; 184/1689 objects degraded (10.894%) 2024-04-22T23:18:45.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:45 smithi069 ceph-mon[63133]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T23:18:45.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:45 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 184/1689 objects degraded (10.894%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:46.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 213510 10.80 MB/sec execute 336 sec latency 2573.495 ms 2024-04-22T23:18:46.900 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:46 smithi069 ceph-mon[63133]: osdmap e96: 8 total, 8 up, 8 in 2024-04-22T23:18:46.900 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:46 smithi069 ceph-mon[63133]: pgmap v116: 97 pgs: 1 remapped+peering, 23 active+undersized+degraded, 14 active+undersized, 59 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 620 KiB/s wr, 26 op/s; 169/1674 objects degraded (10.096%) 2024-04-22T23:18:46.900 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:46.900 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:18:46.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:46 smithi149 ceph-mon[49797]: osdmap e96: 8 total, 8 up, 8 in 2024-04-22T23:18:46.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:46 smithi149 ceph-mon[49797]: pgmap v116: 97 pgs: 1 remapped+peering, 23 active+undersized+degraded, 14 active+undersized, 59 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 620 KiB/s wr, 26 op/s; 169/1674 objects degraded (10.096%) 2024-04-22T23:18:46.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:46.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:18:47.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 213660 10.78 MB/sec execute 337 sec latency 488.666 ms 2024-04-22T23:18:48.044 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:47 smithi069 ceph-mon[63133]: osdmap e97: 8 total, 8 up, 8 in 2024-04-22T23:18:48.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:47 smithi149 ceph-mon[49797]: osdmap e97: 8 total, 8 up, 8 in 2024-04-22T23:18:48.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 213760 10.75 MB/sec execute 338 sec latency 1025.830 ms 2024-04-22T23:18:48.819 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:48 smithi069 ceph-mon[63133]: pgmap v118: 97 pgs: 1 peering, 1 active+recovery_wait+degraded, 89 active+clean, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 2.6 MiB/s wr, 260 op/s; 82/1500 objects degraded (5.467%); 444 KiB/s, 72 keys/s, 28 objects/s recovering 2024-04-22T23:18:48.819 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:48 smithi069 ceph-mon[63133]: osdmap e98: 8 total, 8 up, 8 in 2024-04-22T23:18:49.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:48 smithi149 ceph-mon[49797]: pgmap v118: 97 pgs: 1 peering, 1 active+recovery_wait+degraded, 89 active+clean, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 2.6 MiB/s wr, 260 op/s; 82/1500 objects degraded (5.467%); 444 KiB/s, 72 keys/s, 28 objects/s recovering 2024-04-22T23:18:49.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:48 smithi149 ceph-mon[49797]: osdmap e98: 8 total, 8 up, 8 in 2024-04-22T23:18:49.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 214066 10.74 MB/sec execute 339 sec latency 1010.222 ms 2024-04-22T23:18:49.861 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: osdmap e99: 8 total, 8 up, 8 in 2024-04-22T23:18:49.861 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:49.861 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:49.861 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:18:49.861 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:18:49.861 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:49.861 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:49.861 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:18:49.861 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:49.861 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:49.861 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:49.862 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T23:18:49.862 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T23:18:49.862 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:49 smithi069 ceph-mon[63133]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-22T23:18:50.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: osdmap e99: 8 total, 8 up, 8 in 2024-04-22T23:18:50.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:50.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:18:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:18:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:18:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:18:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T23:18:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T23:18:50.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:49 smithi149 ceph-mon[49797]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-22T23:18:50.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 214377 10.72 MB/sec execute 340 sec latency 715.586 ms 2024-04-22T23:18:50.872 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:50 smithi069 ceph-mon[63133]: pgmap v121: 97 pgs: 1 peering, 1 active+recovery_wait+degraded, 89 active+clean, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 2.0 MiB/s wr, 235 op/s; 82/1500 objects degraded (5.467%); 445 KiB/s, 72 keys/s, 28 objects/s recovering 2024-04-22T23:18:50.873 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:50 smithi069 ceph-mon[63133]: osdmap e100: 8 total, 8 up, 8 in 2024-04-22T23:18:50.873 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:50 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 82/1500 objects degraded (5.467%), 5 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:50 smithi149 ceph-mon[49797]: pgmap v121: 97 pgs: 1 peering, 1 active+recovery_wait+degraded, 89 active+clean, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 2.0 MiB/s wr, 235 op/s; 82/1500 objects degraded (5.467%); 445 KiB/s, 72 keys/s, 28 objects/s recovering 2024-04-22T23:18:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:50 smithi149 ceph-mon[49797]: osdmap e100: 8 total, 8 up, 8 in 2024-04-22T23:18:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:50 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 82/1500 objects degraded (5.467%), 5 pgs degraded (PG_DEGRADED) 2024-04-22T23:18:51.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 214829 10.71 MB/sec execute 341 sec latency 40.376 ms 2024-04-22T23:18:51.812 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:18:52.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 215232 10.71 MB/sec execute 342 sec latency 47.255 ms 2024-04-22T23:18:52.773 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (5m) 11s ago 14m 22.6M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (109s) 11s ago 14m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (103s) 98s ago 13m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (4m) 11s ago 13m 54.1M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (8m) 11s ago 8m 38.5M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (8m) 98s ago 8m 14.6M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (4m) 11s ago 15m 503M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (3m) 98s ago 13m 450M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (3m) 11s ago 15m 48.6M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (2m) 98s ago 13m 34.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (7m) 11s ago 7m 264M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (7m) 98s ago 7m 86.5M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:18:52.774 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (4m) 11s ago 13m 17.4M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:18:52.775 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (4m) 98s ago 13m 16.6M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:18:52.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (68s) 11s ago 12m 253M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:18:52.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (21s) 11s ago 12m 12.6M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:18:52.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (11m) 11s ago 11m 536M 1936M 17.2.0 e1d6a67b021e b4006a135b97 2024-04-22T23:18:52.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (11m) 11s ago 11m 366M 1936M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:18:52.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (10m) 98s ago 10m 547M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:18:52.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (10m) 98s ago 10m 398M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:18:52.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (10m) 98s ago 10m 569M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:18:52.775 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (9m) 98s ago 9m 476M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:18:52.775 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (3m) 11s ago 13m 73.7M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:18:52.921 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:52 smithi069 ceph-mon[63133]: pgmap v123: 97 pgs: 1 peering, 95 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.8 MiB/s wr, 380 op/s; 15/1539 objects degraded (0.975%); 0 B/s, 60 keys/s, 28 objects/s recovering 2024-04-22T23:18:52.921 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:52 smithi069 ceph-mon[63133]: from='client.44179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:52.921 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:52 smithi069 ceph-mon[63133]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:53.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:52 smithi149 ceph-mon[49797]: pgmap v123: 97 pgs: 1 peering, 95 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.8 MiB/s wr, 380 op/s; 15/1539 objects degraded (0.975%); 0 B/s, 60 keys/s, 28 objects/s recovering 2024-04-22T23:18:53.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:52 smithi149 ceph-mon[49797]: from='client.44179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:53.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:52 smithi149 ceph-mon[49797]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:53.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 215669 10.73 MB/sec execute 343 sec latency 38.517 ms 2024-04-22T23:18:53.342 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:18:53.343 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:18:53.343 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:18:53.343 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:18:53.343 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:18:53.343 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:18:53.343 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:18:53.343 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:18:53.343 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-22T23:18:53.343 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:18:53.343 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:18:53.344 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:18:53.344 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:18:53.344 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:18:53.344 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:18:53.344 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-22T23:18:53.344 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 6 2024-04-22T23:18:53.344 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:18:53.344 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:18:53.863 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:18:53.863 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:18:53.863 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:18:53.863 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:18:53.863 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:18:53.863 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:18:53.863 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:18:53.863 INFO:teuthology.orchestra.run.smithi069.stdout: "mon" 2024-04-22T23:18:53.863 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:18:53.864 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "8/23 daemons upgraded", 2024-04-22T23:18:53.864 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T23:18:53.864 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:18:53.864 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:18:53.937 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:53 smithi069 ceph-mon[63133]: from='client.44185 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:53.937 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:53 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/89163864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:54.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:53 smithi149 ceph-mon[49797]: from='client.44185 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:54.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:53 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/89163864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:18:54.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 216037 10.73 MB/sec execute 344 sec latency 51.619 ms 2024-04-22T23:18:54.394 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:18:54.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:54 smithi069 ceph-mon[63133]: pgmap v124: 97 pgs: 97 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.5 MiB/s wr, 324 op/s; 783 KiB/s, 18 objects/s recovering 2024-04-22T23:18:54.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:54 smithi069 ceph-mon[63133]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/1539 objects degraded (0.975%), 1 pg degraded) 2024-04-22T23:18:54.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:54 smithi069 ceph-mon[63133]: Cluster is now healthy 2024-04-22T23:18:54.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:54 smithi069 ceph-mon[63133]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:54.947 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:54 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/1841767954' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:18:55.101 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:54 smithi149 ceph-mon[49797]: pgmap v124: 97 pgs: 97 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.5 MiB/s wr, 324 op/s; 783 KiB/s, 18 objects/s recovering 2024-04-22T23:18:55.101 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:54 smithi149 ceph-mon[49797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/1539 objects degraded (0.975%), 1 pg degraded) 2024-04-22T23:18:55.101 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:54 smithi149 ceph-mon[49797]: Cluster is now healthy 2024-04-22T23:18:55.101 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:54 smithi149 ceph-mon[49797]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:18:55.101 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:54 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/1841767954' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:18:55.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 216480 10.76 MB/sec execute 345 sec latency 32.950 ms 2024-04-22T23:18:56.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 216831 10.75 MB/sec execute 346 sec latency 37.298 ms 2024-04-22T23:18:56.974 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:56 smithi069 ceph-mon[63133]: pgmap v125: 97 pgs: 97 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.4 MiB/s wr, 267 op/s; 593 KiB/s, 14 objects/s recovering 2024-04-22T23:18:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:56 smithi149 ceph-mon[49797]: pgmap v125: 97 pgs: 97 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.4 MiB/s wr, 267 op/s; 593 KiB/s, 14 objects/s recovering 2024-04-22T23:18:57.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 217191 10.73 MB/sec execute 347 sec latency 55.352 ms 2024-04-22T23:18:58.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 217544 10.72 MB/sec execute 348 sec latency 51.364 ms 2024-04-22T23:18:58.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:18:58 smithi069 ceph-mon[63133]: pgmap v126: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.5 MiB/s wr, 555 op/s; 527 KiB/s, 12 objects/s recovering 2024-04-22T23:18:59.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:18:58 smithi149 ceph-mon[49797]: pgmap v126: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.5 MiB/s wr, 555 op/s; 527 KiB/s, 12 objects/s recovering 2024-04-22T23:18:59.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 217967 10.72 MB/sec execute 349 sec latency 38.155 ms 2024-04-22T23:19:00.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 218421 10.72 MB/sec execute 350 sec latency 39.909 ms 2024-04-22T23:19:01.027 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:00 smithi069 ceph-mon[63133]: pgmap v127: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.9 MiB/s wr, 495 op/s; 470 KiB/s, 11 objects/s recovering 2024-04-22T23:19:01.028 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:00 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:01.028 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:00 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:19:01.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:00 smithi149 ceph-mon[49797]: pgmap v127: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.9 MiB/s wr, 495 op/s; 470 KiB/s, 11 objects/s recovering 2024-04-22T23:19:01.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:00 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:01.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:00 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:19:01.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 218846 10.71 MB/sec execute 351 sec latency 38.167 ms 2024-04-22T23:19:02.043 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:02.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:02.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 219313 10.73 MB/sec execute 352 sec latency 33.459 ms 2024-04-22T23:19:03.080 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:02 smithi069 ceph-mon[63133]: pgmap v128: 97 pgs: 97 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 614 op/s; 394 KiB/s, 9 objects/s recovering 2024-04-22T23:19:03.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:02 smithi149 ceph-mon[49797]: pgmap v128: 97 pgs: 97 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 614 op/s; 394 KiB/s, 9 objects/s recovering 2024-04-22T23:19:03.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 219738 10.75 MB/sec execute 353 sec latency 32.884 ms 2024-04-22T23:19:04.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 220135 10.76 MB/sec execute 354 sec latency 56.553 ms 2024-04-22T23:19:05.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:04 smithi069 ceph-mon[63133]: pgmap v129: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 498 op/s; 341 KiB/s, 1 objects/s recovering 2024-04-22T23:19:05.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:04 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T23:19:05.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:04 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T23:19:05.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:04 smithi069 ceph-mon[63133]: Upgrade: osd.2 is safe to restart 2024-04-22T23:19:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:04 smithi149 ceph-mon[49797]: pgmap v129: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 498 op/s; 341 KiB/s, 1 objects/s recovering 2024-04-22T23:19:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:04 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T23:19:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:04 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T23:19:05.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:04 smithi149 ceph-mon[49797]: Upgrade: osd.2 is safe to restart 2024-04-22T23:19:05.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 220527 10.75 MB/sec execute 355 sec latency 48.581 ms 2024-04-22T23:19:06.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 220914 10.74 MB/sec execute 356 sec latency 32.745 ms 2024-04-22T23:19:06.914 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:06 smithi069 ceph-mon[63133]: pgmap v130: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 506 op/s 2024-04-22T23:19:07.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:06 smithi149 ceph-mon[49797]: pgmap v130: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 506 op/s 2024-04-22T23:19:07.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 221248 10.72 MB/sec execute 357 sec latency 69.834 ms 2024-04-22T23:19:08.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 221541 10.72 MB/sec execute 358 sec latency 66.600 ms 2024-04-22T23:19:09.209 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:09 smithi069 ceph-mon[63133]: pgmap v131: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 721 op/s 2024-04-22T23:19:09.209 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:09 smithi069 ceph-mon[63133]: Upgrade: Updating osd.2 2024-04-22T23:19:09.209 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:09.209 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T23:19:09.209 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:19:09.209 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:09 smithi069 ceph-mon[63133]: Deploying daemon osd.2 on smithi069 2024-04-22T23:19:09.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 221977 10.71 MB/sec execute 359 sec latency 48.432 ms 2024-04-22T23:19:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:09 smithi149 ceph-mon[49797]: pgmap v131: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 721 op/s 2024-04-22T23:19:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:09 smithi149 ceph-mon[49797]: Upgrade: Updating osd.2 2024-04-22T23:19:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T23:19:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:19:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:09 smithi149 ceph-mon[49797]: Deploying daemon osd.2 on smithi069 2024-04-22T23:19:10.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 222376 10.71 MB/sec execute 360 sec latency 37.067 ms 2024-04-22T23:19:11.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:11 smithi069 ceph-mon[63133]: pgmap v132: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.8 MiB/s wr, 484 op/s 2024-04-22T23:19:11.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 222743 10.71 MB/sec execute 361 sec latency 40.259 ms 2024-04-22T23:19:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:11 smithi149 ceph-mon[49797]: pgmap v132: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.8 MiB/s wr, 484 op/s 2024-04-22T23:19:12.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 223104 10.73 MB/sec execute 362 sec latency 37.848 ms 2024-04-22T23:19:13.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 223467 10.74 MB/sec execute 363 sec latency 38.545 ms 2024-04-22T23:19:13.314 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:13 smithi069 ceph-mon[63133]: pgmap v133: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 703 op/s 2024-04-22T23:19:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:13 smithi149 ceph-mon[49797]: pgmap v133: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 703 op/s 2024-04-22T23:19:14.073 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:14 smithi069 systemd[1]: Stopping Ceph osd.2 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:19:14.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 223788 10.73 MB/sec execute 364 sec latency 62.307 ms 2024-04-22T23:19:14.576 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:14 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2[40079]: 2024-04-22T23:19:14.368+0000 7fbbc54f4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:19:14.576 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:14 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2[40079]: 2024-04-22T23:19:14.368+0000 7fbbc54f4700 -1 osd.2 100 *** Got signal Terminated *** 2024-04-22T23:19:14.576 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:14 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2[40079]: 2024-04-22T23:19:14.368+0000 7fbbc54f4700 -1 osd.2 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:19:15.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 224072 10.72 MB/sec execute 365 sec latency 97.681 ms 2024-04-22T23:19:15.330 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:15 smithi069 ceph-mon[63133]: pgmap v134: 97 pgs: 97 active+clean; 333 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 514 op/s 2024-04-22T23:19:15.331 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:15 smithi069 ceph-mon[63133]: osd.2 marked itself down and dead 2024-04-22T23:19:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:15 smithi149 ceph-mon[49797]: pgmap v134: 97 pgs: 97 active+clean; 333 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 514 op/s 2024-04-22T23:19:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:15 smithi149 ceph-mon[49797]: osd.2 marked itself down and dead 2024-04-22T23:19:15.991 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:15 smithi069 podman[73988]: 2024-04-22 23:19:15.863496555 +0000 UTC m=+1.635397813 container died b4006a135b972c3dd7d402137eb5389434fe1cc1e5fad19d74a7d301170c0973 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2, io.openshift.tags=base centos centos-stream, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, ceph=True, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, vcs-type=git, maintainer=Guillaume Abrioux ) 2024-04-22T23:19:16.242 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:16 smithi069 ceph-mon[63133]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:19:16.242 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:16 smithi069 ceph-mon[63133]: osdmap e101: 8 total, 7 up, 8 in 2024-04-22T23:19:16.242 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:16.242 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:19:16.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 224324 10.70 MB/sec execute 366 sec latency 1036.057 ms 2024-04-22T23:19:16.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:16 smithi149 ceph-mon[49797]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:19:16.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:16 smithi149 ceph-mon[49797]: osdmap e101: 8 total, 7 up, 8 in 2024-04-22T23:19:16.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:16.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:19:16.529 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:16 smithi069 podman[73988]: 2024-04-22 23:19:16.443968684 +0000 UTC m=+2.215869925 container cleanup b4006a135b972c3dd7d402137eb5389434fe1cc1e5fad19d74a7d301170c0973 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, com.redhat.component=centos-stream-container, ceph=True, architecture=x86_64, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, distribution-scope=public, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.openshift.expose-services=) 2024-04-22T23:19:16.530 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:16 smithi069 bash[73988]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2 2024-04-22T23:19:17.046 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:16 smithi069 podman[74003]: 2024-04-22 23:19:16.796964136 +0000 UTC m=+0.928851051 container remove b4006a135b972c3dd7d402137eb5389434fe1cc1e5fad19d74a7d301170c0973 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2, version=8, ceph=True, release=754, maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.buildah.version=1.19.8, name=centos-stream, architecture=x86_64, GIT_CLEAN=True, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, vcs-type=git, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-22T23:19:17.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 224531 10.68 MB/sec execute 367 sec latency 76.777 ms 2024-04-22T23:19:17.299 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:17 smithi069 podman[74094]: 2024-04-22 23:19:17.176973802 +0000 UTC m=+0.020744006 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:19:17.299 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:17 smithi069 ceph-mon[63133]: pgmap v136: 97 pgs: 5 stale+active+clean, 92 active+clean; 336 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.6 MiB/s wr, 566 op/s 2024-04-22T23:19:17.299 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:17 smithi069 ceph-mon[63133]: osdmap e102: 8 total, 7 up, 8 in 2024-04-22T23:19:17.551 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:17 smithi069 podman[74094]: 2024-04-22 23:19:17.306524203 +0000 UTC m=+0.150294390 container create 7406c89e3fb472451f786556f8a440e88e466cf83844bdad265f2171b313b613 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-22T23:19:17.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:17 smithi149 ceph-mon[49797]: pgmap v136: 97 pgs: 5 stale+active+clean, 92 active+clean; 336 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.6 MiB/s wr, 566 op/s 2024-04-22T23:19:17.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:17 smithi149 ceph-mon[49797]: osdmap e102: 8 total, 7 up, 8 in 2024-04-22T23:19:17.827 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:17 smithi069 podman[74094]: 2024-04-22 23:19:17.670332259 +0000 UTC m=+0.514102448 container init 7406c89e3fb472451f786556f8a440e88e466cf83844bdad265f2171b313b613 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-22T23:19:17.827 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:17 smithi069 podman[74094]: 2024-04-22 23:19:17.680058922 +0000 UTC m=+0.523829114 container start 7406c89e3fb472451f786556f8a440e88e466cf83844bdad265f2171b313b613 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2) 2024-04-22T23:19:17.827 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:17 smithi069 podman[74094]: 2024-04-22 23:19:17.778072789 +0000 UTC m=+0.621842982 container attach 7406c89e3fb472451f786556f8a440e88e466cf83844bdad265f2171b313b613 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T23:19:18.088 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:18 smithi069 podman[74094]: 2024-04-22 23:19:18.058677406 +0000 UTC m=+0.902447589 container died 7406c89e3fb472451f786556f8a440e88e466cf83844bdad265f2171b313b613 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-22T23:19:18.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 224767 10.67 MB/sec execute 368 sec latency 64.327 ms 2024-04-22T23:19:18.343 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:18 smithi069 ceph-mon[63133]: Health check failed: Degraded data redundancy: 144/1701 objects degraded (8.466%), 18 pgs degraded (PG_DEGRADED) 2024-04-22T23:19:18.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:18 smithi149 ceph-mon[49797]: Health check failed: Degraded data redundancy: 144/1701 objects degraded (8.466%), 18 pgs degraded (PG_DEGRADED) 2024-04-22T23:19:19.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 224997 10.65 MB/sec execute 369 sec latency 79.876 ms 2024-04-22T23:19:19.393 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:19 smithi069 ceph-mon[63133]: pgmap v138: 97 pgs: 5 active+undersized, 18 active+undersized+degraded, 74 active+clean; 345 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.4 MiB/s wr, 620 op/s; 144/1701 objects degraded (8.466%) 2024-04-22T23:19:19.394 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:19 smithi069 podman[74119]: 2024-04-22 23:19:19.291064852 +0000 UTC m=+1.226126568 container remove 7406c89e3fb472451f786556f8a440e88e466cf83844bdad265f2171b313b613 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, ceph=True, org.label-schema.build-date=20240416) 2024-04-22T23:19:19.394 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:19 smithi069 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-22T23:19:19.394 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:19 smithi069 systemd[1]: Stopped Ceph osd.2 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:19:19.394 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:19 smithi069 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.2.service: Consumed 3min 87ms CPU time. 2024-04-22T23:19:19.646 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:19 smithi069 systemd[1]: Starting Ceph osd.2 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:19:19.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:19 smithi149 ceph-mon[49797]: pgmap v138: 97 pgs: 5 active+undersized, 18 active+undersized+degraded, 74 active+clean; 345 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.4 MiB/s wr, 620 op/s; 144/1701 objects degraded (8.466%) 2024-04-22T23:19:19.898 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:19 smithi069 podman[74254]: 2024-04-22 23:19:19.748203158 +0000 UTC m=+0.021797349 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:19:20.149 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:19 smithi069 podman[74254]: 2024-04-22 23:19:19.898108902 +0000 UTC m=+0.171703068 container create 6511eaa92717553306b19f7167518452fe591f1135c3596915bf1d20ad010c0b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5) 2024-04-22T23:19:20.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 225268 10.64 MB/sec execute 370 sec latency 51.135 ms 2024-04-22T23:19:20.436 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 podman[74254]: 2024-04-22 23:19:20.299674781 +0000 UTC m=+0.573268967 container init 6511eaa92717553306b19f7167518452fe591f1135c3596915bf1d20ad010c0b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-22T23:19:20.436 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 podman[74254]: 2024-04-22 23:19:20.307554932 +0000 UTC m=+0.581149108 container start 6511eaa92717553306b19f7167518452fe591f1135c3596915bf1d20ad010c0b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-22T23:19:20.436 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 podman[74254]: 2024-04-22 23:19:20.419910161 +0000 UTC m=+0.693504333 container attach 6511eaa92717553306b19f7167518452fe591f1135c3596915bf1d20ad010c0b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T23:19:20.946 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate[74269]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:19:20.946 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate[74269]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:19:20.946 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 bash[74254]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:19:20.946 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 bash[74254]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:19:20.946 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 bash[74254]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T23:19:20.946 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 bash[74254]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 bash[74254]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 bash[74254]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 bash[74254]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 bash[74254]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 bash[74254]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate[74269]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate[74269]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate[74269]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate[74269]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate[74269]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate[74269]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate[74269]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-22T23:19:20.947 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:20 smithi069 podman[74254]: 2024-04-22 23:19:20.873828963 +0000 UTC m=+1.147423135 container died 6511eaa92717553306b19f7167518452fe591f1135c3596915bf1d20ad010c0b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True) 2024-04-22T23:19:21.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 225561 10.63 MB/sec execute 371 sec latency 92.716 ms 2024-04-22T23:19:21.448 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:21 smithi069 ceph-mon[63133]: pgmap v139: 97 pgs: 5 active+undersized, 18 active+undersized+degraded, 74 active+clean; 345 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.5 MiB/s wr, 291 op/s; 144/1701 objects degraded (8.466%) 2024-04-22T23:19:21.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:21 smithi149 ceph-mon[49797]: pgmap v139: 97 pgs: 5 active+undersized, 18 active+undersized+degraded, 74 active+clean; 345 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.5 MiB/s wr, 291 op/s; 144/1701 objects degraded (8.466%) 2024-04-22T23:19:21.956 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:21 smithi069 podman[74397]: 2024-04-22 23:19:21.745826213 +0000 UTC m=+0.866827154 container remove 6511eaa92717553306b19f7167518452fe591f1135c3596915bf1d20ad010c0b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T23:19:22.209 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:22 smithi069 podman[74456]: 2024-04-22 23:19:21.919278234 +0000 UTC m=+0.027951375 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:19:22.209 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:22 smithi069 podman[74456]: 2024-04-22 23:19:22.129302796 +0000 UTC m=+0.237975924 container create 8d3d66dfd7f957e24e15f7d538f1aa7aa063c4c1448e3f6a9037732ac10e94c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:19:22.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 226096 10.65 MB/sec execute 372 sec latency 38.185 ms 2024-04-22T23:19:22.973 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:22 smithi069 podman[74456]: 2024-04-22 23:19:22.750627243 +0000 UTC m=+0.859300373 container init 8d3d66dfd7f957e24e15f7d538f1aa7aa063c4c1448e3f6a9037732ac10e94c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.schema-version=1.0) 2024-04-22T23:19:22.973 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:22 smithi069 podman[74456]: 2024-04-22 23:19:22.756185028 +0000 UTC m=+0.864858161 container start 8d3d66dfd7f957e24e15f7d538f1aa7aa063c4c1448e3f6a9037732ac10e94c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-22T23:19:22.973 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:22 smithi069 ceph-osd[74473]: -- 172.21.15.69:0/778576060 <== mon.0 v2:172.21.15.69:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c781c05680 con 0x55c781bcec00 2024-04-22T23:19:22.973 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:22 smithi069 bash[74456]: 8d3d66dfd7f957e24e15f7d538f1aa7aa063c4c1448e3f6a9037732ac10e94c8 2024-04-22T23:19:22.973 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:22 smithi069 systemd[1]: Started Ceph osd.2 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:19:23.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 226564 10.66 MB/sec execute 373 sec latency 75.785 ms 2024-04-22T23:19:23.477 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:23 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2[74469]: 2024-04-22T23:19:23.330+0000 7f1ba23cf7c0 -1 Falling back to public interface 2024-04-22T23:19:23.478 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:23 smithi069 ceph-mon[63133]: pgmap v140: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 501 op/s; 145/1560 objects degraded (9.295%) 2024-04-22T23:19:23.478 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:23 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:23.478 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:23 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:23.478 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:23 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:19:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:23 smithi149 ceph-mon[49797]: pgmap v140: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 501 op/s; 145/1560 objects degraded (9.295%) 2024-04-22T23:19:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:23 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:23 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:23 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:19:24.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 226874 10.66 MB/sec execute 374 sec latency 42.858 ms 2024-04-22T23:19:24.488 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:24 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 142/1563 objects degraded (9.085%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T23:19:24.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:24 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 142/1563 objects degraded (9.085%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T23:19:24.902 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:19:25.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 227278 10.67 MB/sec execute 375 sec latency 47.011 ms 2024-04-22T23:19:25.533 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:25 smithi069 ceph-mon[63133]: pgmap v141: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 330 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.5 MiB/s wr, 507 op/s; 142/1563 objects degraded (9.085%) 2024-04-22T23:19:25.601 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:25 smithi149 ceph-mon[49797]: pgmap v141: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 330 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.5 MiB/s wr, 507 op/s; 142/1563 objects degraded (9.085%) 2024-04-22T23:19:25.893 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:19:25.893 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (5m) 45s ago 14m 22.6M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:19:25.893 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (2m) 45s ago 14m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:19:25.893 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (2m) 2m ago 14m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (5m) 45s ago 14m 54.1M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (8m) 45s ago 8m 38.5M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (8m) 2m ago 8m 14.6M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (5m) 45s ago 16m 503M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (4m) 2m ago 14m 450M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (3m) 45s ago 16m 48.6M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (3m) 2m ago 13m 34.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (8m) 45s ago 8m 264M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (8m) 2m ago 8m 86.5M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (5m) 45s ago 14m 17.4M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (5m) 2m ago 13m 16.6M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (101s) 45s ago 13m 253M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (54s) 45s ago 12m 12.6M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:19:25.894 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 starting - - - 1936M 2024-04-22T23:19:25.895 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (11m) 45s ago 11m 366M 1936M 17.2.0 e1d6a67b021e ad97ce0d29d4 2024-04-22T23:19:25.895 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (11m) 2m ago 11m 547M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:19:25.895 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (11m) 2m ago 11m 398M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:19:25.895 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (10m) 2m ago 10m 569M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:19:25.895 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (10m) 2m ago 10m 476M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:19:25.895 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (4m) 45s ago 13m 73.7M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:19:26.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 227641 10.66 MB/sec execute 376 sec latency 37.590 ms 2024-04-22T23:19:26.513 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:19:26.513 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:19:26.513 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:19:26.513 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:19:26.513 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:19:26.513 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:19:26.513 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:19:26.513 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:19:26.514 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-22T23:19:26.514 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:19:26.514 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:19:26.514 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:19:26.514 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:19:26.514 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:19:26.514 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:19:26.514 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-22T23:19:26.514 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 6 2024-04-22T23:19:26.514 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:19:26.514 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:19:26.550 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:26 smithi069 ceph-mon[63133]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:19:26.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:26 smithi149 ceph-mon[49797]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:19:27.092 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:19:27.093 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:19:27.093 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:19:27.093 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:19:27.093 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:19:27.093 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:19:27.093 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:19:27.093 INFO:teuthology.orchestra.run.smithi069.stdout: "mon" 2024-04-22T23:19:27.093 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:19:27.093 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "8/23 daemons upgraded", 2024-04-22T23:19:27.093 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T23:19:27.093 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:19:27.093 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:19:27.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 227971 10.65 MB/sec execute 377 sec latency 70.569 ms 2024-04-22T23:19:27.553 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:27 smithi069 ceph-mon[63133]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:19:27.554 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:27 smithi069 ceph-mon[63133]: pgmap v142: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 452 op/s; 142/1572 objects degraded (9.033%) 2024-04-22T23:19:27.554 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:27 smithi069 ceph-mon[63133]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:19:27.554 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:27 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/2975728869' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:19:27.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:27 smithi149 ceph-mon[49797]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:19:27.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:27 smithi149 ceph-mon[49797]: pgmap v142: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 452 op/s; 142/1572 objects degraded (9.033%) 2024-04-22T23:19:27.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:27 smithi149 ceph-mon[49797]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:19:27.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:27 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/2975728869' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:19:27.770 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 142/1572 objects degraded (9.033%), 19 pgs degraded 2024-04-22T23:19:27.770 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-22T23:19:27.770 INFO:teuthology.orchestra.run.smithi069.stdout: osd.2 (root=default,host=smithi069) is down 2024-04-22T23:19:27.770 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 142/1572 objects degraded (9.033%), 19 pgs degraded 2024-04-22T23:19:27.770 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-04-22T23:19:27.771 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-04-22T23:19:27.772 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-04-22T23:19:27.772 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-04-22T23:19:27.772 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-04-22T23:19:28.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228300 10.64 MB/sec execute 378 sec latency 44.091 ms 2024-04-22T23:19:28.585 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:28 smithi069 ceph-mon[63133]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:19:28.586 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:28 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/466234372' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:19:28.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:28 smithi149 ceph-mon[49797]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:19:28.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:28 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/466234372' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:19:29.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228610 10.62 MB/sec execute 379 sec latency 44.483 ms 2024-04-22T23:19:29.341 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:29 smithi069 ceph-mon[63133]: pgmap v143: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.2 MiB/s wr, 624 op/s; 148/1707 objects degraded (8.670%) 2024-04-22T23:19:29.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:29 smithi149 ceph-mon[49797]: pgmap v143: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.2 MiB/s wr, 624 op/s; 148/1707 objects degraded (8.670%) 2024-04-22T23:19:30.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228948 10.62 MB/sec execute 380 sec latency 80.241 ms 2024-04-22T23:19:30.379 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:30 smithi069 ceph-mon[63133]: pgmap v144: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 429 op/s; 148/1707 objects degraded (8.670%) 2024-04-22T23:19:30.379 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:30 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 148/1707 objects degraded (8.670%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T23:19:30.630 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:30 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2[74469]: 2024-04-22T23:19:30.615+0000 7f1ba23cf7c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-22T23:19:30.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:30 smithi149 ceph-mon[49797]: pgmap v144: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 429 op/s; 148/1707 objects degraded (8.670%) 2024-04-22T23:19:30.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:30 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 148/1707 objects degraded (8.670%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T23:19:31.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 229221 10.61 MB/sec execute 381 sec latency 58.295 ms 2024-04-22T23:19:31.667 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:31 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2[74469]: 2024-04-22T23:19:31.622+0000 7f1ba23cf7c0 -1 osd.2 100 log_to_monitors true 2024-04-22T23:19:31.931 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:31.931 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:19:31.931 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:31.931 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:31 smithi069 ceph-mon[63133]: from='osd.2 [v2:172.21.15.69:6818/768758304,v1:172.21.15.69:6819/768758304]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T23:19:32.194 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:19:31 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2[74469]: 2024-04-22T23:19:31.929+0000 7f1b93827700 -1 osd.2 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:19:32.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:32.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:19:32.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:32.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:31 smithi149 ceph-mon[49797]: from='osd.2 [v2:172.21.15.69:6818/768758304,v1:172.21.15.69:6819/768758304]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T23:19:32.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 229506 10.60 MB/sec execute 382 sec latency 45.872 ms 2024-04-22T23:19:33.005 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:32 smithi069 ceph-mon[63133]: pgmap v145: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 327 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 146/1524 objects degraded (9.580%) 2024-04-22T23:19:33.005 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:32 smithi069 ceph-mon[63133]: from='osd.2 [v2:172.21.15.69:6818/768758304,v1:172.21.15.69:6819/768758304]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T23:19:33.005 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:32 smithi069 ceph-mon[63133]: osdmap e103: 8 total, 7 up, 8 in 2024-04-22T23:19:33.005 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:32 smithi069 ceph-mon[63133]: from='osd.2 [v2:172.21.15.69:6818/768758304,v1:172.21.15.69:6819/768758304]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:19:33.005 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:32 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:33.005 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:32 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:32 smithi149 ceph-mon[49797]: pgmap v145: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 327 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 146/1524 objects degraded (9.580%) 2024-04-22T23:19:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:32 smithi149 ceph-mon[49797]: from='osd.2 [v2:172.21.15.69:6818/768758304,v1:172.21.15.69:6819/768758304]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T23:19:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:32 smithi149 ceph-mon[49797]: osdmap e103: 8 total, 7 up, 8 in 2024-04-22T23:19:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:32 smithi149 ceph-mon[49797]: from='osd.2 [v2:172.21.15.69:6818/768758304,v1:172.21.15.69:6819/768758304]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:19:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:32 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:33.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:32 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:33.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 229722 10.60 MB/sec execute 383 sec latency 356.095 ms 2024-04-22T23:19:34.184 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:33 smithi069 ceph-mon[63133]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:19:34.184 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:33 smithi069 ceph-mon[63133]: osd.2 [v2:172.21.15.69:6818/768758304,v1:172.21.15.69:6819/768758304] boot 2024-04-22T23:19:34.184 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:33 smithi069 ceph-mon[63133]: osdmap e104: 8 total, 8 up, 8 in 2024-04-22T23:19:34.184 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:33 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:19:34.184 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:33 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:34.184 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:33 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:34.184 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:33 smithi069 ceph-mon[63133]: osdmap e105: 8 total, 8 up, 8 in 2024-04-22T23:19:34.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:33 smithi149 ceph-mon[49797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:19:34.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:33 smithi149 ceph-mon[49797]: osd.2 [v2:172.21.15.69:6818/768758304,v1:172.21.15.69:6819/768758304] boot 2024-04-22T23:19:34.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:33 smithi149 ceph-mon[49797]: osdmap e104: 8 total, 8 up, 8 in 2024-04-22T23:19:34.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:33 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T23:19:34.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:33 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:34.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:33 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:34.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:33 smithi149 ceph-mon[49797]: osdmap e105: 8 total, 8 up, 8 in 2024-04-22T23:19:34.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 229730 10.57 MB/sec execute 384 sec latency 1333.659 ms 2024-04-22T23:19:35.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 229775 10.55 MB/sec execute 385 sec latency 1977.831 ms 2024-04-22T23:19:35.429 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:35 smithi069 ceph-mon[63133]: pgmap v148: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.8 MiB/s wr, 603 op/s; 148/1515 objects degraded (9.769%) 2024-04-22T23:19:35.429 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:35 smithi069 ceph-mon[63133]: osdmap e106: 8 total, 8 up, 8 in 2024-04-22T23:19:35.429 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:35 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 148/1515 objects degraded (9.769%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T23:19:35.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:35 smithi149 ceph-mon[49797]: pgmap v148: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.8 MiB/s wr, 603 op/s; 148/1515 objects degraded (9.769%) 2024-04-22T23:19:35.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:35 smithi149 ceph-mon[49797]: osdmap e106: 8 total, 8 up, 8 in 2024-04-22T23:19:35.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:35 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 148/1515 objects degraded (9.769%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T23:19:36.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 230112 10.55 MB/sec execute 386 sec latency 81.481 ms 2024-04-22T23:19:37.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 230373 10.54 MB/sec execute 387 sec latency 261.589 ms 2024-04-22T23:19:37.504 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:37 smithi069 ceph-mon[63133]: pgmap v151: 97 pgs: 1 remapped+peering, 5 active+undersized, 18 active+undersized+degraded, 73 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.7 MiB/s wr, 249 op/s; 138/1512 objects degraded (9.127%) 2024-04-22T23:19:37.505 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:37 smithi069 ceph-mon[63133]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T23:19:37.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:37 smithi149 ceph-mon[49797]: pgmap v151: 97 pgs: 1 remapped+peering, 5 active+undersized, 18 active+undersized+degraded, 73 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.7 MiB/s wr, 249 op/s; 138/1512 objects degraded (9.127%) 2024-04-22T23:19:37.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:37 smithi149 ceph-mon[49797]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T23:19:38.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 230635 10.53 MB/sec execute 388 sec latency 971.878 ms 2024-04-22T23:19:38.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:38 smithi069 ceph-mon[63133]: osdmap e108: 8 total, 8 up, 8 in 2024-04-22T23:19:38.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:38.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:38.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:19:38.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:19:38.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:38.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:38.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:19:38.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:19:38.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:19:38.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:19:38.512 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T23:19:38.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:38 smithi149 ceph-mon[49797]: osdmap e108: 8 total, 8 up, 8 in 2024-04-22T23:19:38.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:38.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:38.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:19:38.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:19:38.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:38.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:38.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:19:38.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:19:38.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:19:38.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:19:38.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T23:19:39.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 230878 10.53 MB/sec execute 389 sec latency 1028.071 ms 2024-04-22T23:19:39.518 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:39 smithi069 ceph-mon[63133]: pgmap v154: 97 pgs: 1 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 92 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.4 MiB/s wr, 305 op/s; 7/1581 objects degraded (0.443%); 183 KiB/s, 37 objects/s recovering 2024-04-22T23:19:39.518 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:39 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T23:19:39.518 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:39 smithi069 ceph-mon[63133]: Upgrade: osd.3 is safe to restart 2024-04-22T23:19:39.518 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:39 smithi069 ceph-mon[63133]: osdmap e109: 8 total, 8 up, 8 in 2024-04-22T23:19:39.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:39 smithi149 ceph-mon[49797]: pgmap v154: 97 pgs: 1 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 92 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.4 MiB/s wr, 305 op/s; 7/1581 objects degraded (0.443%); 183 KiB/s, 37 objects/s recovering 2024-04-22T23:19:39.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:39 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T23:19:39.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:39 smithi149 ceph-mon[49797]: Upgrade: osd.3 is safe to restart 2024-04-22T23:19:39.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:39 smithi149 ceph-mon[49797]: osdmap e109: 8 total, 8 up, 8 in 2024-04-22T23:19:40.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 231114 10.52 MB/sec execute 390 sec latency 772.347 ms 2024-04-22T23:19:40.536 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:40 smithi069 ceph-mon[63133]: osdmap e110: 8 total, 8 up, 8 in 2024-04-22T23:19:40.536 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:40 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 7/1581 objects degraded (0.443%), 1 pg degraded (PG_DEGRADED) 2024-04-22T23:19:40.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:40 smithi149 ceph-mon[49797]: osdmap e110: 8 total, 8 up, 8 in 2024-04-22T23:19:40.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:40 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 7/1581 objects degraded (0.443%), 1 pg degraded (PG_DEGRADED) 2024-04-22T23:19:41.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 231378 10.51 MB/sec execute 391 sec latency 61.698 ms 2024-04-22T23:19:41.559 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:41 smithi069 ceph-mon[63133]: pgmap v157: 97 pgs: 1 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 92 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.9 MiB/s wr, 281 op/s; 7/1581 objects degraded (0.443%); 183 KiB/s, 37 objects/s recovering 2024-04-22T23:19:41.559 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:41 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:41.559 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:41 smithi069 ceph-mon[63133]: Upgrade: Updating osd.3 2024-04-22T23:19:41.559 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:41 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:41.559 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:41 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T23:19:41.559 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:41 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:19:41.559 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:41 smithi069 ceph-mon[63133]: Deploying daemon osd.3 on smithi069 2024-04-22T23:19:41.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:41 smithi149 ceph-mon[49797]: pgmap v157: 97 pgs: 1 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 92 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.9 MiB/s wr, 281 op/s; 7/1581 objects degraded (0.443%); 183 KiB/s, 37 objects/s recovering 2024-04-22T23:19:41.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:41 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:41.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:41 smithi149 ceph-mon[49797]: Upgrade: Updating osd.3 2024-04-22T23:19:41.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:41 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:41.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:41 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T23:19:41.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:41 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:19:41.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:41 smithi149 ceph-mon[49797]: Deploying daemon osd.3 on smithi069 2024-04-22T23:19:42.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 231605 10.49 MB/sec execute 392 sec latency 96.194 ms 2024-04-22T23:19:42.578 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:42 smithi069 ceph-mon[63133]: pgmap v158: 97 pgs: 1 active+undersized+remapped, 96 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 5.7 MiB/s wr, 484 op/s; 138 KiB/s, 34 objects/s recovering 2024-04-22T23:19:42.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:42 smithi149 ceph-mon[49797]: pgmap v158: 97 pgs: 1 active+undersized+remapped, 96 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 5.7 MiB/s wr, 484 op/s; 138 KiB/s, 34 objects/s recovering 2024-04-22T23:19:43.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 231905 10.48 MB/sec execute 393 sec latency 52.031 ms 2024-04-22T23:19:43.595 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:43 smithi069 ceph-mon[63133]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/1581 objects degraded (0.443%), 1 pg degraded) 2024-04-22T23:19:43.595 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:43 smithi069 ceph-mon[63133]: Cluster is now healthy 2024-04-22T23:19:43.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:43 smithi149 ceph-mon[49797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/1581 objects degraded (0.443%), 1 pg degraded) 2024-04-22T23:19:43.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:43 smithi149 ceph-mon[49797]: Cluster is now healthy 2024-04-22T23:19:44.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 232157 10.46 MB/sec execute 394 sec latency 50.047 ms 2024-04-22T23:19:44.391 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:44 smithi069 ceph-mon[63133]: pgmap v159: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.5 MiB/s wr, 314 op/s; 3.1 KiB/s, 5 objects/s recovering 2024-04-22T23:19:44.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:44 smithi149 ceph-mon[49797]: pgmap v159: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.5 MiB/s wr, 314 op/s; 3.1 KiB/s, 5 objects/s recovering 2024-04-22T23:19:45.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 232527 10.46 MB/sec execute 395 sec latency 39.363 ms 2024-04-22T23:19:46.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 232984 10.47 MB/sec execute 396 sec latency 39.632 ms 2024-04-22T23:19:46.931 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:46 smithi069 ceph-mon[63133]: pgmap v160: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.2 MiB/s wr, 272 op/s; 2.5 KiB/s, 4 objects/s recovering 2024-04-22T23:19:46.932 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:46.932 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:19:47.187 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:47 smithi069 systemd[1]: Stopping Ceph osd.3 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:19:47.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:46 smithi149 ceph-mon[49797]: pgmap v160: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.2 MiB/s wr, 272 op/s; 2.5 KiB/s, 4 objects/s recovering 2024-04-22T23:19:47.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:47.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:19:47.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 233371 10.47 MB/sec execute 397 sec latency 58.062 ms 2024-04-22T23:19:47.691 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:47 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3[43345]: 2024-04-22T23:19:47.460+0000 7f5feb07d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:19:47.691 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:47 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3[43345]: 2024-04-22T23:19:47.460+0000 7f5feb07d700 -1 osd.3 110 *** Got signal Terminated *** 2024-04-22T23:19:47.691 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:47 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3[43345]: 2024-04-22T23:19:47.460+0000 7f5feb07d700 -1 osd.3 110 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:19:47.941 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:47 smithi069 ceph-mon[63133]: osd.3 marked itself down and dead 2024-04-22T23:19:48.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:47 smithi149 ceph-mon[49797]: osd.3 marked itself down and dead 2024-04-22T23:19:48.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 233630 10.47 MB/sec execute 398 sec latency 483.267 ms 2024-04-22T23:19:48.818 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:48 smithi069 podman[76555]: 2024-04-22 23:19:48.489223219 +0000 UTC m=+1.189164958 container died ad97ce0d29d47d7de7370eb6883ff83c35fa1289c89e4d052771d721a164cf03 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3, release=754, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, version=8, io.openshift.expose-services=, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-22T23:19:49.068 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:48 smithi069 podman[76555]: 2024-04-22 23:19:48.859950021 +0000 UTC m=+1.559891766 container cleanup ad97ce0d29d47d7de7370eb6883ff83c35fa1289c89e4d052771d721a164cf03 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3, vcs-type=git, maintainer=Guillaume Abrioux , architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, RELEASE=HEAD, ceph=True, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_CLEAN=True) 2024-04-22T23:19:49.069 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:48 smithi069 bash[76555]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3 2024-04-22T23:19:49.069 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:48 smithi069 ceph-mon[63133]: pgmap v161: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 484 op/s; 2.1 KiB/s, 3 objects/s recovering 2024-04-22T23:19:49.069 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:48 smithi069 ceph-mon[63133]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:19:49.069 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:48 smithi069 ceph-mon[63133]: osdmap e111: 8 total, 7 up, 8 in 2024-04-22T23:19:49.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:48 smithi149 ceph-mon[49797]: pgmap v161: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 484 op/s; 2.1 KiB/s, 3 objects/s recovering 2024-04-22T23:19:49.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:48 smithi149 ceph-mon[49797]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:19:49.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:48 smithi149 ceph-mon[49797]: osdmap e111: 8 total, 7 up, 8 in 2024-04-22T23:19:49.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 233789 10.45 MB/sec execute 399 sec latency 1028.885 ms 2024-04-22T23:19:49.320 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:49 smithi069 podman[76572]: 2024-04-22 23:19:49.093092076 +0000 UTC m=+0.602618018 container remove ad97ce0d29d47d7de7370eb6883ff83c35fa1289c89e4d052771d721a164cf03 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3, release=754, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, name=centos-stream, architecture=x86_64, GIT_BRANCH=HEAD, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, ceph=True, RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.openshift.expose-services=, version=8, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870) 2024-04-22T23:19:49.823 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:49 smithi069 podman[76650]: 2024-04-22 23:19:49.556970518 +0000 UTC m=+0.027578718 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:19:49.824 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:49 smithi069 podman[76650]: 2024-04-22 23:19:49.744346543 +0000 UTC m=+0.214954767 container create 17235107907f312498f8bb0ca87cea011fb6f7d6db33f6774b34d2fc8b5473a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, ceph=True, CEPH_POINT_RELEASE=) 2024-04-22T23:19:50.078 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:49 smithi069 ceph-mon[63133]: osdmap e112: 8 total, 7 up, 8 in 2024-04-22T23:19:50.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:49 smithi149 ceph-mon[49797]: osdmap e112: 8 total, 7 up, 8 in 2024-04-22T23:19:50.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 234122 10.45 MB/sec execute 400 sec latency 37.676 ms 2024-04-22T23:19:50.329 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:50 smithi069 podman[76650]: 2024-04-22 23:19:50.156644417 +0000 UTC m=+0.627252588 container init 17235107907f312498f8bb0ca87cea011fb6f7d6db33f6774b34d2fc8b5473a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-22T23:19:50.330 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:50 smithi069 podman[76650]: 2024-04-22 23:19:50.167694355 +0000 UTC m=+0.638302522 container start 17235107907f312498f8bb0ca87cea011fb6f7d6db33f6774b34d2fc8b5473a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-22T23:19:50.586 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:50 smithi069 podman[76650]: 2024-04-22 23:19:50.335664974 +0000 UTC m=+0.806273133 container attach 17235107907f312498f8bb0ca87cea011fb6f7d6db33f6774b34d2fc8b5473a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-deactivate, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2) 2024-04-22T23:19:50.855 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:50 smithi069 podman[76650]: 2024-04-22 23:19:50.648227073 +0000 UTC m=+1.118835233 container died 17235107907f312498f8bb0ca87cea011fb6f7d6db33f6774b34d2fc8b5473a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-22T23:19:50.856 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:50 smithi069 ceph-mon[63133]: pgmap v164: 97 pgs: 16 stale+active+clean, 81 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.1 MiB/s wr, 381 op/s 2024-04-22T23:19:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:50 smithi149 ceph-mon[49797]: pgmap v164: 97 pgs: 16 stale+active+clean, 81 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.1 MiB/s wr, 381 op/s 2024-04-22T23:19:51.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 234469 10.45 MB/sec execute 401 sec latency 41.764 ms 2024-04-22T23:19:51.617 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:51 smithi069 podman[76675]: 2024-04-22 23:19:51.614982509 +0000 UTC m=+0.958262869 container remove 17235107907f312498f8bb0ca87cea011fb6f7d6db33f6774b34d2fc8b5473a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T23:19:51.868 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:51 smithi069 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-22T23:19:51.868 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:51 smithi069 systemd[1]: Stopped Ceph osd.3 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:19:51.868 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:51 smithi069 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.3.service: Consumed 4min 23.754s CPU time. 2024-04-22T23:19:52.130 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:52.130 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:51 smithi069 systemd[1]: Starting Ceph osd.3 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:19:52.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:52.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 234787 10.45 MB/sec execute 402 sec latency 54.673 ms 2024-04-22T23:19:52.386 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:52 smithi069 podman[76773]: 2024-04-22 23:19:52.124737776 +0000 UTC m=+0.024039741 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:19:52.386 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:52 smithi069 podman[76773]: 2024-04-22 23:19:52.310087744 +0000 UTC m=+0.209389707 container create c989840c26a99d2776fcb987c91b550e9ad9ea3354fec14c45489aa78c14b655 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:19:52.887 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:52 smithi069 podman[76773]: 2024-04-22 23:19:52.68235159 +0000 UTC m=+0.581653550 container init c989840c26a99d2776fcb987c91b550e9ad9ea3354fec14c45489aa78c14b655 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240416) 2024-04-22T23:19:52.888 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:52 smithi069 podman[76773]: 2024-04-22 23:19:52.688672109 +0000 UTC m=+0.587974067 container start c989840c26a99d2776fcb987c91b550e9ad9ea3354fec14c45489aa78c14b655 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-22T23:19:52.888 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:52 smithi069 podman[76773]: 2024-04-22 23:19:52.866629218 +0000 UTC m=+0.765931189 container attach c989840c26a99d2776fcb987c91b550e9ad9ea3354fec14c45489aa78c14b655 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416) 2024-04-22T23:19:53.143 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:52 smithi069 ceph-mon[63133]: pgmap v165: 97 pgs: 9 active+undersized, 17 active+undersized+degraded, 6 stale+active+clean, 65 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.6 MiB/s wr, 406 op/s; 162/1632 objects degraded (9.926%) 2024-04-22T23:19:53.144 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:52 smithi069 ceph-mon[63133]: Health check failed: Degraded data redundancy: 162/1632 objects degraded (9.926%), 17 pgs degraded (PG_DEGRADED) 2024-04-22T23:19:53.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:52 smithi149 ceph-mon[49797]: pgmap v165: 97 pgs: 9 active+undersized, 17 active+undersized+degraded, 6 stale+active+clean, 65 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.6 MiB/s wr, 406 op/s; 162/1632 objects degraded (9.926%) 2024-04-22T23:19:53.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:52 smithi149 ceph-mon[49797]: Health check failed: Degraded data redundancy: 162/1632 objects degraded (9.926%), 17 pgs degraded (PG_DEGRADED) 2024-04-22T23:19:53.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 235091 10.45 MB/sec execute 403 sec latency 53.141 ms 2024-04-22T23:19:53.395 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate[76789]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:19:53.395 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate[76789]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:19:53.395 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 bash[76773]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:19:53.395 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 bash[76773]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:19:53.395 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 bash[76773]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T23:19:53.395 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 bash[76773]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T23:19:53.395 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 bash[76773]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T23:19:53.395 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 bash[76773]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T23:19:53.395 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 bash[76773]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-22T23:19:53.395 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 bash[76773]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T23:19:53.396 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 bash[76773]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-22T23:19:53.396 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate[76789]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T23:19:53.396 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate[76789]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T23:19:53.396 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate[76789]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T23:19:53.396 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate[76789]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T23:19:53.396 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate[76789]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-22T23:19:53.396 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate[76789]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T23:19:53.396 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate[76789]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-22T23:19:53.396 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:53 smithi069 podman[76773]: 2024-04-22 23:19:53.387031852 +0000 UTC m=+1.286333807 container died c989840c26a99d2776fcb987c91b550e9ad9ea3354fec14c45489aa78c14b655 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2) 2024-04-22T23:19:54.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 235428 10.43 MB/sec execute 404 sec latency 60.883 ms 2024-04-22T23:19:54.561 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:54 smithi069 podman[76932]: 2024-04-22 23:19:54.550889361 +0000 UTC m=+1.149606702 container remove c989840c26a99d2776fcb987c91b550e9ad9ea3354fec14c45489aa78c14b655 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:19:55.033 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:54 smithi069 ceph-mon[63133]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.1 MiB/s wr, 547 op/s; 213/1641 objects degraded (12.980%) 2024-04-22T23:19:55.034 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:54 smithi069 podman[76990]: 2024-04-22 23:19:54.722879329 +0000 UTC m=+0.029815163 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:19:55.034 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:54 smithi069 podman[76990]: 2024-04-22 23:19:54.902525911 +0000 UTC m=+0.209461715 container create e7772e3632e1b859e51662440c2b33fc6b478eac536381721b7b6fb8a9aaeaad (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416) 2024-04-22T23:19:55.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:54 smithi149 ceph-mon[49797]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.1 MiB/s wr, 547 op/s; 213/1641 objects degraded (12.980%) 2024-04-22T23:19:55.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 235749 10.42 MB/sec execute 405 sec latency 49.843 ms 2024-04-22T23:19:55.800 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:55 smithi069 podman[76990]: 2024-04-22 23:19:55.548629727 +0000 UTC m=+0.855565539 container init e7772e3632e1b859e51662440c2b33fc6b478eac536381721b7b6fb8a9aaeaad (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS) 2024-04-22T23:19:55.801 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:55 smithi069 podman[76990]: 2024-04-22 23:19:55.553551464 +0000 UTC m=+0.860487271 container start e7772e3632e1b859e51662440c2b33fc6b478eac536381721b7b6fb8a9aaeaad (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD) 2024-04-22T23:19:55.801 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:55 smithi069 ceph-osd[77007]: -- 172.21.15.69:0/3636284991 <== mon.0 v2:172.21.15.69:3300/0 3 ==== mon_map magic: 0 ==== 357+0+0 (secure 0 0 0) 0x55ebc1fb3ba0 con 0x55ebc1f85000 2024-04-22T23:19:55.801 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:55 smithi069 ceph-osd[77007]: -- 172.21.15.69:0/3636284991 <== mon.0 v2:172.21.15.69:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ebc1fbb680 con 0x55ebc1f85000 2024-04-22T23:19:55.801 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:55 smithi069 bash[76990]: e7772e3632e1b859e51662440c2b33fc6b478eac536381721b7b6fb8a9aaeaad 2024-04-22T23:19:55.801 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:55 smithi069 systemd[1]: Started Ceph osd.3 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:19:56.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 236042 10.41 MB/sec execute 406 sec latency 56.751 ms 2024-04-22T23:19:56.307 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:19:56 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3[77003]: 2024-04-22T23:19:56.122+0000 7f7964b907c0 -1 Falling back to public interface 2024-04-22T23:19:56.813 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:56 smithi069 ceph-mon[63133]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.4 MiB/s wr, 273 op/s; 212/1629 objects degraded (13.014%) 2024-04-22T23:19:56.814 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:56.814 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:56.814 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:19:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:56 smithi149 ceph-mon[49797]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.4 MiB/s wr, 273 op/s; 212/1629 objects degraded (13.014%) 2024-04-22T23:19:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:19:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:19:57.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 236438 10.42 MB/sec execute 407 sec latency 44.013 ms 2024-04-22T23:19:58.089 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:57 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 207/1605 objects degraded (12.897%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T23:19:58.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:57 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 207/1605 objects degraded (12.897%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T23:19:58.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 236852 10.42 MB/sec execute 408 sec latency 46.950 ms 2024-04-22T23:19:58.299 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:19:59.094 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:58 smithi069 ceph-mon[63133]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.6 MiB/s wr, 449 op/s; 207/1605 objects degraded (12.897%) 2024-04-22T23:19:59.094 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:58 smithi069 ceph-mon[63133]: from='client.34304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:19:59.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:58 smithi149 ceph-mon[49797]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.6 MiB/s wr, 449 op/s; 207/1605 objects degraded (12.897%) 2024-04-22T23:19:59.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:58 smithi149 ceph-mon[49797]: from='client.34304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:19:59.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 237286 10.43 MB/sec execute 409 sec latency 38.139 ms 2024-04-22T23:19:59.364 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:19:59.364 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (6m) 27s ago 15m 22.7M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:19:59.364 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (2m) 27s ago 15m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (2m) 2m ago 14m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (6m) 27s ago 15m 53.7M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (9m) 27s ago 9m 38.5M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (9m) 2m ago 9m 14.6M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (5m) 27s ago 16m 507M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (4m) 2m ago 14m 450M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (4m) 27s ago 16m 49.8M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (3m) 2m ago 14m 34.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (9m) 27s ago 9m 264M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (8m) 2m ago 8m 86.5M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (5m) 27s ago 14m 20.0M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (5m) 2m ago 14m 16.6M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (2m) 27s ago 13m 313M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:19:59.365 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (87s) 27s ago 13m 241M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:19:59.366 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (37s) 27s ago 12m 15.3M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:19:59.366 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 starting - - - 1936M 2024-04-22T23:19:59.366 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (12m) 2m ago 12m 547M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:19:59.366 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (11m) 2m ago 11m 398M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:19:59.366 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (11m) 2m ago 11m 569M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:19:59.366 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (10m) 2m ago 10m 476M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:19:59.366 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (4m) 27s ago 14m 73.9M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:19:59.982 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:19:59.983 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:19:59.983 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:19:59.983 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:19:59.983 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:19:59.983 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:19:59.983 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:19:59.983 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:19:59.983 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-22T23:19:59.983 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 3 2024-04-22T23:19:59.983 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:19:59.983 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:19:59.983 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:19:59.984 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:19:59.984 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:19:59.984 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-22T23:19:59.984 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 7 2024-04-22T23:19:59.984 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:19:59.984 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:20:00.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:19:59 smithi069 ceph-mon[63133]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:00.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:19:59 smithi149 ceph-mon[49797]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:00.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 237652 10.43 MB/sec execute 410 sec latency 63.274 ms 2024-04-22T23:20:00.514 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:20:00.514 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:20:00.514 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:20:00.515 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:20:00.515 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:20:00.515 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:20:00.515 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:20:00.515 INFO:teuthology.orchestra.run.smithi069.stdout: "mon" 2024-04-22T23:20:00.515 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:20:00.515 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "9/23 daemons upgraded", 2024-04-22T23:20:00.515 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T23:20:00.515 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:20:00.515 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:20:00.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.2 MiB/s wr, 408 op/s; 207/1605 objects degraded (12.897%) 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/54486380' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 207/1605 objects degraded (12.897%), 27 pgs degraded 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: [WRN] OSD_DOWN: 1 osds down 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: osd.3 (root=default,host=smithi069) is down 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: [WRN] PG_DEGRADED: Degraded data redundancy: 207/1605 objects degraded (12.897%), 27 pgs degraded 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 2.a is active+undersized+degraded, acting [1,7] 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 2.d is active+undersized+degraded, acting [1,4] 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 2.e is active+undersized+degraded, acting [2,7] 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-04-22T23:20:00.887 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 2.1b is active+undersized+degraded, acting [2,6] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.b is active+undersized+degraded, acting [0,4] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.c is active+undersized+degraded, acting [5,6] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-04-22T23:20:00.888 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-04-22T23:20:00.889 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-04-22T23:20:00.889 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-04-22T23:20:00.889 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-04-22T23:20:00.889 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-04-22T23:20:00.889 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-04-22T23:20:00.889 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:00.889 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:00 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:20:01.154 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 207/1605 objects degraded (12.897%), 27 pgs degraded 2024-04-22T23:20:01.154 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-22T23:20:01.154 INFO:teuthology.orchestra.run.smithi069.stdout: osd.3 (root=default,host=smithi069) is down 2024-04-22T23:20:01.154 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 207/1605 objects degraded (12.897%), 27 pgs degraded 2024-04-22T23:20:01.154 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-04-22T23:20:01.154 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-04-22T23:20:01.154 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-04-22T23:20:01.154 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-04-22T23:20:01.154 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1b is active+undersized+degraded, acting [2,6] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-04-22T23:20:01.155 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-04-22T23:20:01.156 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-04-22T23:20:01.156 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-04-22T23:20:01.156 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-04-22T23:20:01.156 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-04-22T23:20:01.156 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-04-22T23:20:01.156 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-04-22T23:20:01.156 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-04-22T23:20:01.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:01.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.2 MiB/s wr, 408 op/s; 207/1605 objects degraded (12.897%) 2024-04-22T23:20:01.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/54486380' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:01.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 207/1605 objects degraded (12.897%), 27 pgs degraded 2024-04-22T23:20:01.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: [WRN] OSD_DOWN: 1 osds down 2024-04-22T23:20:01.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: osd.3 (root=default,host=smithi069) is down 2024-04-22T23:20:01.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: [WRN] PG_DEGRADED: Degraded data redundancy: 207/1605 objects degraded (12.897%), 27 pgs degraded 2024-04-22T23:20:01.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-04-22T23:20:01.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-04-22T23:20:01.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 2.a is active+undersized+degraded, acting [1,7] 2024-04-22T23:20:01.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 2.d is active+undersized+degraded, acting [1,4] 2024-04-22T23:20:01.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 2.e is active+undersized+degraded, acting [2,7] 2024-04-22T23:20:01.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-04-22T23:20:01.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-04-22T23:20:01.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-04-22T23:20:01.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-04-22T23:20:01.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 2.1b is active+undersized+degraded, acting [2,6] 2024-04-22T23:20:01.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-04-22T23:20:01.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-04-22T23:20:01.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-04-22T23:20:01.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.b is active+undersized+degraded, acting [0,4] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.c is active+undersized+degraded, acting [5,6] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-04-22T23:20:01.203 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:01.204 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:00 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:20:01.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 238022 10.43 MB/sec execute 411 sec latency 60.044 ms 2024-04-22T23:20:01.890 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:01 smithi069 ceph-mon[63133]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:01.890 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:01 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/399245774' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:20:02.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:01 smithi149 ceph-mon[49797]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:02.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:01 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/399245774' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:20:02.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 238425 10.44 MB/sec execute 412 sec latency 39.238 ms 2024-04-22T23:20:02.900 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:02 smithi069 ceph-mon[63133]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 6.0 MiB/s wr, 570 op/s; 210/1647 objects degraded (12.750%) 2024-04-22T23:20:03.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:02 smithi149 ceph-mon[49797]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 6.0 MiB/s wr, 570 op/s; 210/1647 objects degraded (12.750%) 2024-04-22T23:20:03.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 238773 10.43 MB/sec execute 413 sec latency 43.293 ms 2024-04-22T23:20:03.914 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:20:03 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3[77003]: 2024-04-22T23:20:03.699+0000 7f7964b907c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-22T23:20:04.172 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:03 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 210/1659 objects degraded (12.658%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:04.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:03 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 210/1659 objects degraded (12.658%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:04.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 239118 10.41 MB/sec execute 414 sec latency 38.194 ms 2024-04-22T23:20:04.938 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:04 smithi069 ceph-mon[63133]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 554 op/s; 210/1659 objects degraded (12.658%) 2024-04-22T23:20:05.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:04 smithi149 ceph-mon[49797]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 554 op/s; 210/1659 objects degraded (12.658%) 2024-04-22T23:20:05.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 239416 10.40 MB/sec execute 415 sec latency 45.023 ms 2024-04-22T23:20:05.448 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:20:05 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3[77003]: 2024-04-22T23:20:05.349+0000 7f7964b907c0 -1 osd.3 110 log_to_monitors true 2024-04-22T23:20:06.209 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:06 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:06.210 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:06 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:06.210 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:06 smithi069 ceph-mon[63133]: from='osd.3 [v2:172.21.15.69:6826/2518302378,v1:172.21.15.69:6827/2518302378]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T23:20:06.210 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:20:06 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3[77003]: 2024-04-22T23:20:06.186+0000 7f7955fe8700 -1 osd.3 110 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:20:06.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 239782 10.41 MB/sec execute 416 sec latency 66.558 ms 2024-04-22T23:20:06.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:06 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:06.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:06 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:06.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:06 smithi149 ceph-mon[49797]: from='osd.3 [v2:172.21.15.69:6826/2518302378,v1:172.21.15.69:6827/2518302378]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T23:20:07.232 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:07 smithi069 ceph-mon[63133]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 463 op/s; 206/1641 objects degraded (12.553%) 2024-04-22T23:20:07.233 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:07 smithi069 ceph-mon[63133]: from='osd.3 [v2:172.21.15.69:6826/2518302378,v1:172.21.15.69:6827/2518302378]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T23:20:07.233 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:07 smithi069 ceph-mon[63133]: osdmap e113: 8 total, 7 up, 8 in 2024-04-22T23:20:07.233 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:07 smithi069 ceph-mon[63133]: from='osd.3 [v2:172.21.15.69:6826/2518302378,v1:172.21.15.69:6827/2518302378]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:20:07.233 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:07 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:07.233 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:07 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:07.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 240106 10.40 MB/sec execute 417 sec latency 171.338 ms 2024-04-22T23:20:07.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:07 smithi149 ceph-mon[49797]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 463 op/s; 206/1641 objects degraded (12.553%) 2024-04-22T23:20:07.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:07 smithi149 ceph-mon[49797]: from='osd.3 [v2:172.21.15.69:6826/2518302378,v1:172.21.15.69:6827/2518302378]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T23:20:07.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:07 smithi149 ceph-mon[49797]: osdmap e113: 8 total, 7 up, 8 in 2024-04-22T23:20:07.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:07 smithi149 ceph-mon[49797]: from='osd.3 [v2:172.21.15.69:6826/2518302378,v1:172.21.15.69:6827/2518302378]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-04-22T23:20:07.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:07 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:07.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:07 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:08.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 240147 10.38 MB/sec execute 418 sec latency 1097.376 ms 2024-04-22T23:20:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:08 smithi069 ceph-mon[63133]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:20:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:08 smithi069 ceph-mon[63133]: osd.3 [v2:172.21.15.69:6826/2518302378,v1:172.21.15.69:6827/2518302378] boot 2024-04-22T23:20:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:08 smithi069 ceph-mon[63133]: osdmap e114: 8 total, 8 up, 8 in 2024-04-22T23:20:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:08 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:20:08.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:08 smithi069 ceph-mon[63133]: osdmap e115: 8 total, 8 up, 8 in 2024-04-22T23:20:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:08 smithi149 ceph-mon[49797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:20:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:08 smithi149 ceph-mon[49797]: osd.3 [v2:172.21.15.69:6826/2518302378,v1:172.21.15.69:6827/2518302378] boot 2024-04-22T23:20:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:08 smithi149 ceph-mon[49797]: osdmap e114: 8 total, 8 up, 8 in 2024-04-22T23:20:08.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:08 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T23:20:08.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:08 smithi149 ceph-mon[49797]: osdmap e115: 8 total, 8 up, 8 in 2024-04-22T23:20:09.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 240234 10.37 MB/sec execute 419 sec latency 904.415 ms 2024-04-22T23:20:09.319 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:09 smithi069 ceph-mon[63133]: pgmap v175: 97 pgs: 6 peering, 12 active+undersized, 23 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 697 op/s; 179/1527 objects degraded (11.722%) 2024-04-22T23:20:09.320 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:09 smithi069 ceph-mon[63133]: osdmap e116: 8 total, 8 up, 8 in 2024-04-22T23:20:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:09 smithi149 ceph-mon[49797]: pgmap v175: 97 pgs: 6 peering, 12 active+undersized, 23 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 697 op/s; 179/1527 objects degraded (11.722%) 2024-04-22T23:20:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:09 smithi149 ceph-mon[49797]: osdmap e116: 8 total, 8 up, 8 in 2024-04-22T23:20:10.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 240588 10.37 MB/sec execute 420 sec latency 915.046 ms 2024-04-22T23:20:10.335 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:10 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 179/1527 objects degraded (11.722%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:10.335 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:10 smithi069 ceph-mon[63133]: osdmap e117: 8 total, 8 up, 8 in 2024-04-22T23:20:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:10 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 179/1527 objects degraded (11.722%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:10 smithi149 ceph-mon[49797]: osdmap e117: 8 total, 8 up, 8 in 2024-04-22T23:20:11.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 240843 10.36 MB/sec execute 421 sec latency 1047.417 ms 2024-04-22T23:20:11.323 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:11 smithi069 ceph-mon[63133]: pgmap v178: 97 pgs: 6 peering, 12 active+undersized, 23 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 623 op/s; 179/1527 objects degraded (11.722%) 2024-04-22T23:20:11.323 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:11 smithi069 ceph-mon[63133]: osdmap e118: 8 total, 8 up, 8 in 2024-04-22T23:20:11.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:11 smithi149 ceph-mon[49797]: pgmap v178: 97 pgs: 6 peering, 12 active+undersized, 23 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 623 op/s; 179/1527 objects degraded (11.722%) 2024-04-22T23:20:11.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:11 smithi149 ceph-mon[49797]: osdmap e118: 8 total, 8 up, 8 in 2024-04-22T23:20:12.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 241116 10.36 MB/sec execute 422 sec latency 934.441 ms 2024-04-22T23:20:13.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: pgmap v181: 97 pgs: 7 peering, 1 active+undersized, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 83 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.0 MiB/s wr, 276 op/s; 87/1608 objects degraded (5.410%); 294 KiB/s, 45 objects/s recovering 2024-04-22T23:20:13.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: osdmap e119: 8 total, 8 up, 8 in 2024-04-22T23:20:13.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:13.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:13.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:20:13.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:20:13.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:13.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:13.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:20:13.133 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:13.133 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:13.133 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:13.133 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T23:20:13.133 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T23:20:13.133 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:13 smithi069 ceph-mon[63133]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-04-22T23:20:13.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 241407 10.35 MB/sec execute 423 sec latency 761.537 ms 2024-04-22T23:20:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: pgmap v181: 97 pgs: 7 peering, 1 active+undersized, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 83 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.0 MiB/s wr, 276 op/s; 87/1608 objects degraded (5.410%); 294 KiB/s, 45 objects/s recovering 2024-04-22T23:20:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: osdmap e119: 8 total, 8 up, 8 in 2024-04-22T23:20:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:20:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:20:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:20:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T23:20:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T23:20:13.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:13 smithi149 ceph-mon[49797]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-04-22T23:20:14.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:14 smithi069 ceph-mon[63133]: osdmap e120: 8 total, 8 up, 8 in 2024-04-22T23:20:14.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 241735 10.34 MB/sec execute 424 sec latency 44.680 ms 2024-04-22T23:20:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:14 smithi149 ceph-mon[49797]: osdmap e120: 8 total, 8 up, 8 in 2024-04-22T23:20:15.166 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:15 smithi069 ceph-mon[63133]: pgmap v184: 97 pgs: 1 peering, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.4 MiB/s wr, 299 op/s; 76/1623 objects degraded (4.683%); 1.3 MiB/s, 50 objects/s recovering 2024-04-22T23:20:15.166 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:15 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 76/1623 objects degraded (4.683%), 3 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:15.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 242062 10.35 MB/sec execute 425 sec latency 50.635 ms 2024-04-22T23:20:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:15 smithi149 ceph-mon[49797]: pgmap v184: 97 pgs: 1 peering, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.4 MiB/s wr, 299 op/s; 76/1623 objects degraded (4.683%); 1.3 MiB/s, 50 objects/s recovering 2024-04-22T23:20:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:15 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 76/1623 objects degraded (4.683%), 3 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:16.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 242329 10.34 MB/sec execute 426 sec latency 80.083 ms 2024-04-22T23:20:17.025 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:16 smithi069 ceph-mon[63133]: pgmap v185: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.8 MiB/s wr, 213 op/s; 29/1635 objects degraded (1.774%); 794 KiB/s, 39 objects/s recovering 2024-04-22T23:20:17.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:17.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:20:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:16 smithi149 ceph-mon[49797]: pgmap v185: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.8 MiB/s wr, 213 op/s; 29/1635 objects degraded (1.774%); 794 KiB/s, 39 objects/s recovering 2024-04-22T23:20:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:20:17.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 242540 10.32 MB/sec execute 427 sec latency 91.652 ms 2024-04-22T23:20:18.046 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:17 smithi069 ceph-mon[63133]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1635 objects degraded (1.774%), 1 pg degraded) 2024-04-22T23:20:18.047 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:17 smithi069 ceph-mon[63133]: Cluster is now healthy 2024-04-22T23:20:18.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:17 smithi149 ceph-mon[49797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1635 objects degraded (1.774%), 1 pg degraded) 2024-04-22T23:20:18.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:17 smithi149 ceph-mon[49797]: Cluster is now healthy 2024-04-22T23:20:18.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 242819 10.30 MB/sec execute 428 sec latency 70.688 ms 2024-04-22T23:20:19.086 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:18 smithi069 ceph-mon[63133]: pgmap v186: 97 pgs: 97 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.2 MiB/s wr, 514 op/s; 770 KiB/s, 41 objects/s recovering 2024-04-22T23:20:19.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:18 smithi149 ceph-mon[49797]: pgmap v186: 97 pgs: 97 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.2 MiB/s wr, 514 op/s; 770 KiB/s, 41 objects/s recovering 2024-04-22T23:20:19.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 243028 10.29 MB/sec execute 429 sec latency 65.002 ms 2024-04-22T23:20:20.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 243220 10.28 MB/sec execute 430 sec latency 48.539 ms 2024-04-22T23:20:20.905 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:20 smithi069 ceph-mon[63133]: pgmap v187: 97 pgs: 97 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.2 MiB/s wr, 312 op/s; 616 KiB/s, 15 objects/s recovering 2024-04-22T23:20:21.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:20 smithi149 ceph-mon[49797]: pgmap v187: 97 pgs: 97 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.2 MiB/s wr, 312 op/s; 616 KiB/s, 15 objects/s recovering 2024-04-22T23:20:21.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 243545 10.28 MB/sec execute 431 sec latency 51.029 ms 2024-04-22T23:20:22.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 243939 10.29 MB/sec execute 432 sec latency 43.469 ms 2024-04-22T23:20:22.943 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:22 smithi069 ceph-mon[63133]: pgmap v188: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.1 MiB/s wr, 415 op/s; 86 KiB/s, 12 objects/s recovering 2024-04-22T23:20:23.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:22 smithi149 ceph-mon[49797]: pgmap v188: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.1 MiB/s wr, 415 op/s; 86 KiB/s, 12 objects/s recovering 2024-04-22T23:20:23.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 244346 10.29 MB/sec execute 433 sec latency 47.753 ms 2024-04-22T23:20:24.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 244580 10.28 MB/sec execute 434 sec latency 59.052 ms 2024-04-22T23:20:24.986 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:24 smithi069 ceph-mon[63133]: pgmap v189: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.9 MiB/s wr, 417 op/s; 76 KiB/s, 9 objects/s recovering 2024-04-22T23:20:25.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:24 smithi149 ceph-mon[49797]: pgmap v189: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.9 MiB/s wr, 417 op/s; 76 KiB/s, 9 objects/s recovering 2024-04-22T23:20:25.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 244993 10.28 MB/sec execute 435 sec latency 35.279 ms 2024-04-22T23:20:26.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 245325 10.27 MB/sec execute 436 sec latency 44.533 ms 2024-04-22T23:20:27.007 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:26 smithi069 ceph-mon[63133]: pgmap v190: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.0 MiB/s wr, 390 op/s; 68 KiB/s, 8 objects/s recovering 2024-04-22T23:20:27.008 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:26 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:26 smithi149 ceph-mon[49797]: pgmap v190: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.0 MiB/s wr, 390 op/s; 68 KiB/s, 8 objects/s recovering 2024-04-22T23:20:27.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:26 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:27.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 245681 10.28 MB/sec execute 437 sec latency 53.950 ms 2024-04-22T23:20:28.018 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:27 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T23:20:28.019 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:27 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T23:20:28.019 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:27 smithi069 ceph-mon[63133]: Upgrade: osd.4 is safe to restart 2024-04-22T23:20:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:27 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T23:20:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:27 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T23:20:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:27 smithi149 ceph-mon[49797]: Upgrade: osd.4 is safe to restart 2024-04-22T23:20:28.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 246061 10.28 MB/sec execute 438 sec latency 46.865 ms 2024-04-22T23:20:29.028 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:28 smithi069 ceph-mon[63133]: pgmap v191: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 564 op/s; 53 KiB/s, 4 objects/s recovering 2024-04-22T23:20:29.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:28 smithi149 ceph-mon[49797]: pgmap v191: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 564 op/s; 53 KiB/s, 4 objects/s recovering 2024-04-22T23:20:29.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 246466 10.26 MB/sec execute 439 sec latency 41.120 ms 2024-04-22T23:20:30.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 246727 10.26 MB/sec execute 440 sec latency 55.736 ms 2024-04-22T23:20:30.552 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:30 smithi069 ceph-mon[63133]: Upgrade: Updating osd.4 2024-04-22T23:20:30.553 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:30.553 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T23:20:30.553 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:20:30.553 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:30 smithi069 ceph-mon[63133]: Deploying daemon osd.4 on smithi149 2024-04-22T23:20:30.553 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:30 smithi069 ceph-mon[63133]: pgmap v192: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 377 op/s 2024-04-22T23:20:30.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:30 smithi149 ceph-mon[49797]: Upgrade: Updating osd.4 2024-04-22T23:20:30.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:30.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T23:20:30.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:20:30.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:30 smithi149 ceph-mon[49797]: Deploying daemon osd.4 on smithi149 2024-04-22T23:20:30.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:30 smithi149 ceph-mon[49797]: pgmap v192: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 377 op/s 2024-04-22T23:20:31.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 247079 10.27 MB/sec execute 441 sec latency 39.702 ms 2024-04-22T23:20:31.697 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:20:31.926 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:31.926 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:20:32.102 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:32.102 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:20:32.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 247439 10.28 MB/sec execute 442 sec latency 70.112 ms 2024-04-22T23:20:32.708 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:20:32.709 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (6m) 27s ago 15m 23.2M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:20:32.709 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (3m) 27s ago 15m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:20:32.709 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (3m) 3m ago 15m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:20:32.709 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (6m) 27s ago 15m 57.1M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:20:32.709 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (9m) 27s ago 9m 38.4M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:20:32.709 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (9m) 3m ago 9m 14.6M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:20:32.709 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (6m) 27s ago 17m 509M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:20:32.709 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (5m) 3m ago 15m 450M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:20:32.709 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (4m) 27s ago 17m 51.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:20:32.709 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (4m) 3m ago 15m 34.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:20:32.709 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (9m) 27s ago 9m 264M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:20:32.709 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (9m) 3m ago 9m 86.5M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:20:32.710 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (6m) 27s ago 15m 17.0M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:20:32.710 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (6m) 3m ago 14m 16.6M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:20:32.710 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (2m) 27s ago 14m 345M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:20:32.710 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (2m) 27s ago 13m 283M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:20:32.710 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (70s) 27s ago 13m 210M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:20:32.710 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (37s) 27s ago 12m 12.7M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 e7772e3632e1 2024-04-22T23:20:32.710 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (12m) 3m ago 12m 547M 4096M 17.2.0 e1d6a67b021e d4343603cf9b 2024-04-22T23:20:32.710 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (12m) 3m ago 12m 398M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:20:32.710 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (11m) 3m ago 11m 569M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:20:32.710 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (11m) 3m ago 11m 476M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:20:32.710 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (5m) 27s ago 14m 73.9M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:20:32.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:32 smithi069 ceph-mon[63133]: from='client.34330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:32.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:32 smithi069 ceph-mon[63133]: pgmap v193: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.2 MiB/s wr, 459 op/s 2024-04-22T23:20:32.948 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:32 smithi069 ceph-mon[63133]: from='client.44247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:32.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:32 smithi149 ceph-mon[49797]: from='client.34330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:32.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:32 smithi149 ceph-mon[49797]: pgmap v193: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.2 MiB/s wr, 459 op/s 2024-04-22T23:20:32.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:32 smithi149 ceph-mon[49797]: from='client.44247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:33.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 247751 10.27 MB/sec execute 443 sec latency 75.618 ms 2024-04-22T23:20:33.345 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:20:33.345 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:20:33.346 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:20:33.346 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:20:33.346 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:20:33.346 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:20:33.346 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:20:33.346 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:20:33.346 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-22T23:20:33.349 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 4 2024-04-22T23:20:33.349 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:20:33.349 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:20:33.349 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:20:33.349 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:20:33.350 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:20:33.350 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-22T23:20:33.350 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 8 2024-04-22T23:20:33.350 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:20:33.350 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:20:33.868 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:20:33.869 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:20:33.869 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:20:33.869 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:20:33.869 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:20:33.869 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:20:33.869 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:20:33.869 INFO:teuthology.orchestra.run.smithi069.stdout: "mon" 2024-04-22T23:20:33.869 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:20:33.869 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "10/23 daemons upgraded", 2024-04-22T23:20:33.869 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T23:20:33.869 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:20:33.869 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:20:33.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:33 smithi149 ceph-mon[49797]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:33.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:33 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/3468342143' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:33.970 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:33 smithi069 ceph-mon[63133]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:33.971 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:33 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/3468342143' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:34.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 247994 10.26 MB/sec execute 444 sec latency 85.120 ms 2024-04-22T23:20:34.381 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:34 smithi149 systemd[1]: Stopping Ceph osd.4 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:20:34.424 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:20:34.700 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[28172]: 2024-04-22T23:20:34.379+0000 7f6db4ec3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:20:34.700 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[28172]: 2024-04-22T23:20:34.379+0000 7f6db4ec3700 -1 osd.4 120 *** Got signal Terminated *** 2024-04-22T23:20:34.700 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[28172]: 2024-04-22T23:20:34.379+0000 7f6db4ec3700 -1 osd.4 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:20:35.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:34 smithi069 ceph-mon[63133]: pgmap v194: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 468 op/s 2024-04-22T23:20:35.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:34 smithi069 ceph-mon[63133]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:35.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:34 smithi069 ceph-mon[63133]: osd.4 marked itself down and dead 2024-04-22T23:20:35.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:34 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/2361426221' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:20:35.152 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:34 smithi149 ceph-mon[49797]: pgmap v194: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 468 op/s 2024-04-22T23:20:35.152 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:34 smithi149 ceph-mon[49797]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:20:35.152 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:34 smithi149 ceph-mon[49797]: osd.4 marked itself down and dead 2024-04-22T23:20:35.152 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:34 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/2361426221' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:20:35.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 248086 10.24 MB/sec execute 445 sec latency 479.937 ms 2024-04-22T23:20:35.763 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:35 smithi149 podman[54255]: 2024-04-22 23:20:35.46050121 +0000 UTC m=+1.204668451 container died d4343603cf9b9194b90e998b6a05030b5e087214538a25e0e87624de63b59a3f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.buildah.version=1.19.8, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=) 2024-04-22T23:20:36.076 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:35 smithi069 ceph-mon[63133]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:20:36.077 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:35 smithi069 ceph-mon[63133]: osdmap e121: 8 total, 7 up, 8 in 2024-04-22T23:20:36.131 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:35 smithi149 ceph-mon[49797]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:20:36.131 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:35 smithi149 ceph-mon[49797]: osdmap e121: 8 total, 7 up, 8 in 2024-04-22T23:20:36.131 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:35 smithi149 podman[54255]: 2024-04-22 23:20:35.862208171 +0000 UTC m=+1.606375417 container cleanup d4343603cf9b9194b90e998b6a05030b5e087214538a25e0e87624de63b59a3f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.component=centos-stream-container, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, version=8, vcs-type=git, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, distribution-scope=public, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, release=754, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-22T23:20:36.131 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:35 smithi149 bash[54255]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4 2024-04-22T23:20:36.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 248156 10.22 MB/sec execute 446 sec latency 1047.667 ms 2024-04-22T23:20:36.450 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:36 smithi149 podman[54279]: 2024-04-22 23:20:36.130569646 +0000 UTC m=+0.667091093 container remove d4343603cf9b9194b90e998b6a05030b5e087214538a25e0e87624de63b59a3f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vcs-type=git, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, ceph=True, RELEASE=HEAD, version=8, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_CLEAN=True, io.openshift.expose-services=) 2024-04-22T23:20:36.844 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:36 smithi069 ceph-mon[63133]: pgmap v196: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 503 op/s 2024-04-22T23:20:36.844 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:36 smithi069 ceph-mon[63133]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T23:20:36.845 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:36 smithi069 ceph-mon[63133]: osdmap e122: 8 total, 7 up, 8 in 2024-04-22T23:20:36.885 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:36 smithi149 ceph-mon[49797]: pgmap v196: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 503 op/s 2024-04-22T23:20:36.885 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:36 smithi149 ceph-mon[49797]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T23:20:36.885 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:36 smithi149 ceph-mon[49797]: osdmap e122: 8 total, 7 up, 8 in 2024-04-22T23:20:36.886 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:36 smithi149 podman[54368]: 2024-04-22 23:20:36.502128943 +0000 UTC m=+0.021073830 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:20:36.886 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:36 smithi149 podman[54368]: 2024-04-22 23:20:36.615128 +0000 UTC m=+0.134072943 container create 3ff1d6db21dc763cbfb27b576526e0c8130f0847dbada49aa56735b747c53d20 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-22T23:20:37.200 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:36 smithi149 podman[54368]: 2024-04-22 23:20:36.884845362 +0000 UTC m=+0.403790235 container init 3ff1d6db21dc763cbfb27b576526e0c8130f0847dbada49aa56735b747c53d20 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux ) 2024-04-22T23:20:37.200 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:36 smithi149 podman[54368]: 2024-04-22 23:20:36.889365823 +0000 UTC m=+0.408310703 container start 3ff1d6db21dc763cbfb27b576526e0c8130f0847dbada49aa56735b747c53d20 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-deactivate, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-22T23:20:37.201 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:37 smithi149 podman[54368]: 2024-04-22 23:20:37.001733656 +0000 UTC m=+0.520678536 container attach 3ff1d6db21dc763cbfb27b576526e0c8130f0847dbada49aa56735b747c53d20 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T23:20:37.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 248445 10.21 MB/sec execute 447 sec latency 89.507 ms 2024-04-22T23:20:37.598 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:37 smithi149 podman[54368]: 2024-04-22 23:20:37.293084213 +0000 UTC m=+0.812029089 container died 3ff1d6db21dc763cbfb27b576526e0c8130f0847dbada49aa56735b747c53d20 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:20:38.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:37 smithi069 ceph-mon[63133]: Health check failed: Degraded data redundancy: 203/1617 objects degraded (12.554%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:38.193 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:37 smithi149 ceph-mon[49797]: Health check failed: Degraded data redundancy: 203/1617 objects degraded (12.554%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:38.194 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:37 smithi149 podman[54394]: 2024-04-22 23:20:37.939838915 +0000 UTC m=+0.641072375 container remove 3ff1d6db21dc763cbfb27b576526e0c8130f0847dbada49aa56735b747c53d20 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416) 2024-04-22T23:20:38.194 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:37 smithi149 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-22T23:20:38.194 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:37 smithi149 systemd[1]: Stopped Ceph osd.4 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:20:38.194 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:37 smithi149 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.4.service: Consumed 4min 42.883s CPU time. 2024-04-22T23:20:38.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 248703 10.20 MB/sec execute 448 sec latency 59.057 ms 2024-04-22T23:20:38.450 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:38 smithi149 systemd[1]: Starting Ceph osd.4 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:20:38.450 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:38 smithi149 podman[54496]: 2024-04-22 23:20:38.344322073 +0000 UTC m=+0.023681546 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:20:38.861 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:38 smithi149 podman[54496]: 2024-04-22 23:20:38.574479554 +0000 UTC m=+0.253839020 container create 11e037e310f07b5179be0e86355bd0b5e27d51d55554bfcb918fe501a0dea3af (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-22T23:20:38.924 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:38 smithi069 ceph-mon[63133]: pgmap v198: 97 pgs: 12 active+undersized, 21 active+undersized+degraded, 8 peering, 56 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 520 op/s; 203/1617 objects degraded (12.554%) 2024-04-22T23:20:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:38 smithi149 ceph-mon[49797]: pgmap v198: 97 pgs: 12 active+undersized, 21 active+undersized+degraded, 8 peering, 56 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 520 op/s; 203/1617 objects degraded (12.554%) 2024-04-22T23:20:39.200 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:38 smithi149 podman[54496]: 2024-04-22 23:20:38.860768697 +0000 UTC m=+0.540128156 container init 11e037e310f07b5179be0e86355bd0b5e27d51d55554bfcb918fe501a0dea3af (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-22T23:20:39.200 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:38 smithi149 podman[54496]: 2024-04-22 23:20:38.865253969 +0000 UTC m=+0.544613432 container start 11e037e310f07b5179be0e86355bd0b5e27d51d55554bfcb918fe501a0dea3af (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-22T23:20:39.201 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:38 smithi149 podman[54496]: 2024-04-22 23:20:38.969715141 +0000 UTC m=+0.649074599 container attach 11e037e310f07b5179be0e86355bd0b5e27d51d55554bfcb918fe501a0dea3af (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-22T23:20:39.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 248977 10.20 MB/sec execute 449 sec latency 53.027 ms 2024-04-22T23:20:39.951 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate[54509]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:20:39.951 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate[54509]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:20:39.951 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 bash[54496]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:20:39.951 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 bash[54496]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:20:39.951 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 bash[54496]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T23:20:39.951 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 bash[54496]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T23:20:39.951 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 bash[54496]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T23:20:39.951 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 bash[54496]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T23:20:39.952 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 bash[54496]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-22T23:20:39.952 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 bash[54496]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T23:20:39.952 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 bash[54496]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-22T23:20:39.952 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate[54509]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T23:20:39.952 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate[54509]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T23:20:39.952 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate[54509]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T23:20:39.952 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate[54509]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T23:20:39.952 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate[54509]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-22T23:20:39.952 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate[54509]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T23:20:39.952 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate[54509]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-22T23:20:39.952 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 conmon[54509]: conmon 11e037e310f07b5179be : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-11e037e310f07b5179be0e86355bd0b5e27d51d55554bfcb918fe501a0dea3af.scope/container/memory.events 2024-04-22T23:20:39.952 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:39 smithi149 podman[54496]: 2024-04-22 23:20:39.693490723 +0000 UTC m=+1.372850188 container died 11e037e310f07b5179be0e86355bd0b5e27d51d55554bfcb918fe501a0dea3af (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_CLEAN=True) 2024-04-22T23:20:40.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 249330 10.21 MB/sec execute 450 sec latency 50.186 ms 2024-04-22T23:20:40.643 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:40 smithi149 podman[54675]: 2024-04-22 23:20:40.383192639 +0000 UTC m=+0.679375348 container remove 11e037e310f07b5179be0e86355bd0b5e27d51d55554bfcb918fe501a0dea3af (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-activate, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T23:20:40.643 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:40 smithi149 podman[54740]: 2024-04-22 23:20:40.51920081 +0000 UTC m=+0.023965647 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:20:40.904 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:40 smithi149 ceph-mon[49797]: pgmap v199: 97 pgs: 12 active+undersized, 21 active+undersized+degraded, 8 peering, 56 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.1 MiB/s wr, 398 op/s; 203/1617 objects degraded (12.554%) 2024-04-22T23:20:40.904 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:40 smithi149 podman[54740]: 2024-04-22 23:20:40.642556667 +0000 UTC m=+0.147321475 container create 036a1a8ba3c977e415b72fd1cb8f50fbcf3e4a5c641c42d45b94757b766c2551 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-22T23:20:40.984 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:40 smithi069 ceph-mon[63133]: pgmap v199: 97 pgs: 12 active+undersized, 21 active+undersized+degraded, 8 peering, 56 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.1 MiB/s wr, 398 op/s; 203/1617 objects degraded (12.554%) 2024-04-22T23:20:41.200 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:40 smithi149 podman[54740]: 2024-04-22 23:20:40.904099697 +0000 UTC m=+0.408864514 container init 036a1a8ba3c977e415b72fd1cb8f50fbcf3e4a5c641c42d45b94757b766c2551 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T23:20:41.200 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:40 smithi149 podman[54740]: 2024-04-22 23:20:40.907407245 +0000 UTC m=+0.412172054 container start 036a1a8ba3c977e415b72fd1cb8f50fbcf3e4a5c641c42d45b94757b766c2551 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-22T23:20:41.200 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:41 smithi149 bash[54740]: 036a1a8ba3c977e415b72fd1cb8f50fbcf3e4a5c641c42d45b94757b766c2551 2024-04-22T23:20:41.201 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:41 smithi149 systemd[1]: Started Ceph osd.4 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:20:41.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 249721 10.20 MB/sec execute 451 sec latency 89.082 ms 2024-04-22T23:20:41.950 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:41 smithi149 ceph-osd[54758]: -- 172.21.15.149:0/433460597 <== mon.1 v2:172.21.15.149:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5595e71f6000 con 0x5595e6412c00 2024-04-22T23:20:42.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:42 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:42.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:42 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:42.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:42 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:20:42.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 250042 10.18 MB/sec execute 452 sec latency 47.714 ms 2024-04-22T23:20:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:42 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:42 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:42 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:20:42.450 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:42 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:20:42.022+0000 7f7cfb4247c0 -1 Falling back to public interface 2024-04-22T23:20:43.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 250271 10.18 MB/sec execute 453 sec latency 46.153 ms 2024-04-22T23:20:43.311 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:43 smithi069 ceph-mon[63133]: pgmap v200: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.1 MiB/s wr, 325 op/s; 259/1611 objects degraded (16.077%) 2024-04-22T23:20:43.311 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:43 smithi069 ceph-mon[63133]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T23:20:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:43 smithi149 ceph-mon[49797]: pgmap v200: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.1 MiB/s wr, 325 op/s; 259/1611 objects degraded (16.077%) 2024-04-22T23:20:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:43 smithi149 ceph-mon[49797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T23:20:44.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 250569 10.18 MB/sec execute 454 sec latency 40.834 ms 2024-04-22T23:20:44.325 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:44 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 264/1623 objects degraded (16.266%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:44.389 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:44 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 264/1623 objects degraded (16.266%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:45.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 250896 10.19 MB/sec execute 455 sec latency 50.595 ms 2024-04-22T23:20:45.424 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:45 smithi069 ceph-mon[63133]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.1 MiB/s wr, 430 op/s; 264/1623 objects degraded (16.266%) 2024-04-22T23:20:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:45 smithi149 ceph-mon[49797]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.1 MiB/s wr, 430 op/s; 264/1623 objects degraded (16.266%) 2024-04-22T23:20:46.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 251141 10.18 MB/sec execute 456 sec latency 87.164 ms 2024-04-22T23:20:46.961 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:46 smithi069 ceph-mon[63133]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 389 op/s; 261/1635 objects degraded (15.963%) 2024-04-22T23:20:46.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:46.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:20:46.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:46.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:47.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:46 smithi149 ceph-mon[49797]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 389 op/s; 261/1635 objects degraded (15.963%) 2024-04-22T23:20:47.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:47.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:20:47.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:47.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:47.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 251424 10.17 MB/sec execute 457 sec latency 75.114 ms 2024-04-22T23:20:48.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 251640 10.16 MB/sec execute 458 sec latency 47.301 ms 2024-04-22T23:20:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:48 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:48 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:48.480 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:48 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:48.480 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:48 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:49.200 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:48 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:20:48.921+0000 7f7cfb4247c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-22T23:20:49.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:49 smithi069 ceph-mon[63133]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 483 op/s; 257/1608 objects degraded (15.983%) 2024-04-22T23:20:49.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 251897 10.15 MB/sec execute 459 sec latency 49.079 ms 2024-04-22T23:20:49.556 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:49 smithi149 ceph-mon[49797]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 483 op/s; 257/1608 objects degraded (15.983%) 2024-04-22T23:20:50.290 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:50 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 257/1608 objects degraded (15.983%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:50.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 252174 10.14 MB/sec execute 460 sec latency 65.655 ms 2024-04-22T23:20:50.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:50 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 257/1608 objects degraded (15.983%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:50.700 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:50 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:20:50.485+0000 7f7cfb4247c0 -1 osd.4 120 log_to_monitors true 2024-04-22T23:20:51.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 252431 10.14 MB/sec execute 461 sec latency 47.669 ms 2024-04-22T23:20:51.317 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:51 smithi069 ceph-mon[63133]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.5 MiB/s wr, 366 op/s; 257/1608 objects degraded (15.983%) 2024-04-22T23:20:51.317 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:51 smithi069 ceph-mon[63133]: from='osd.4 [v2:172.21.15.149:6800/3315340692,v1:172.21.15.149:6801/3315340692]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T23:20:51.317 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:51 smithi069 ceph-mon[63133]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T23:20:51.317 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:51.317 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:51.317 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:20:51.317 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:20:51.317 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:51.317 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:51.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:51 smithi149 ceph-mon[49797]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.5 MiB/s wr, 366 op/s; 257/1608 objects degraded (15.983%) 2024-04-22T23:20:51.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:51 smithi149 ceph-mon[49797]: from='osd.4 [v2:172.21.15.149:6800/3315340692,v1:172.21.15.149:6801/3315340692]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T23:20:51.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:51 smithi149 ceph-mon[49797]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T23:20:51.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:51.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:51.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:20:51.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:20:51.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:51.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:20:52.200 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:20:51 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:20:51.796+0000 7f7ce8874700 -1 osd.4 120 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:20:52.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 252619 10.13 MB/sec execute 462 sec latency 161.950 ms 2024-04-22T23:20:52.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:52 smithi069 ceph-mon[63133]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T23:20:52.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:52 smithi069 ceph-mon[63133]: osdmap e123: 8 total, 7 up, 8 in 2024-04-22T23:20:52.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:52 smithi069 ceph-mon[63133]: from='osd.4 [v2:172.21.15.149:6800/3315340692,v1:172.21.15.149:6801/3315340692]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:20:52.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:52 smithi069 ceph-mon[63133]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:20:52.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:52 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:20:52.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:52 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:52.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:52 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:52.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:52 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:52.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:52 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T23:20:52.372 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:52 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T23:20:52.372 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:52 smithi069 ceph-mon[63133]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-22T23:20:52.372 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:52 smithi069 ceph-mon[63133]: from='osd.4 ' entity='osd.4' 2024-04-22T23:20:52.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:52 smithi149 ceph-mon[49797]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T23:20:52.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:52 smithi149 ceph-mon[49797]: osdmap e123: 8 total, 7 up, 8 in 2024-04-22T23:20:52.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:52 smithi149 ceph-mon[49797]: from='osd.4 [v2:172.21.15.149:6800/3315340692,v1:172.21.15.149:6801/3315340692]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:20:52.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:52 smithi149 ceph-mon[49797]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:20:52.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:52 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:20:52.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:52 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:52.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:52 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:52.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:52 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:20:52.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:52 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T23:20:52.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:52 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T23:20:52.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:52 smithi149 ceph-mon[49797]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-22T23:20:52.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:52 smithi149 ceph-mon[49797]: from='osd.4 ' entity='osd.4' 2024-04-22T23:20:53.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 252847 10.13 MB/sec execute 463 sec latency 493.393 ms 2024-04-22T23:20:53.392 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:53 smithi069 ceph-mon[63133]: pgmap v206: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.3 MiB/s wr, 381 op/s; 260/1650 objects degraded (15.758%) 2024-04-22T23:20:53.393 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:53 smithi069 ceph-mon[63133]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:20:53.393 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:53 smithi069 ceph-mon[63133]: osd.4 [v2:172.21.15.149:6800/3315340692,v1:172.21.15.149:6801/3315340692] boot 2024-04-22T23:20:53.393 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:53 smithi069 ceph-mon[63133]: osdmap e124: 8 total, 8 up, 8 in 2024-04-22T23:20:53.393 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:53 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:20:53.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:53 smithi149 ceph-mon[49797]: pgmap v206: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.3 MiB/s wr, 381 op/s; 260/1650 objects degraded (15.758%) 2024-04-22T23:20:53.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:53 smithi149 ceph-mon[49797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:20:53.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:53 smithi149 ceph-mon[49797]: osd.4 [v2:172.21.15.149:6800/3315340692,v1:172.21.15.149:6801/3315340692] boot 2024-04-22T23:20:53.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:53 smithi149 ceph-mon[49797]: osdmap e124: 8 total, 8 up, 8 in 2024-04-22T23:20:53.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:53 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T23:20:54.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 252987 10.11 MB/sec execute 464 sec latency 1077.238 ms 2024-04-22T23:20:54.918 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:54 smithi069 ceph-mon[63133]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.9 MiB/s wr, 487 op/s; 243/1602 objects degraded (15.169%) 2024-04-22T23:20:54.919 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:54 smithi069 ceph-mon[63133]: osdmap e125: 8 total, 8 up, 8 in 2024-04-22T23:20:55.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:54 smithi149 ceph-mon[49797]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.9 MiB/s wr, 487 op/s; 243/1602 objects degraded (15.169%) 2024-04-22T23:20:55.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:54 smithi149 ceph-mon[49797]: osdmap e125: 8 total, 8 up, 8 in 2024-04-22T23:20:55.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 253234 10.09 MB/sec execute 465 sec latency 1185.434 ms 2024-04-22T23:20:55.934 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:55 smithi069 ceph-mon[63133]: osdmap e126: 8 total, 8 up, 8 in 2024-04-22T23:20:55.935 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:55 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 243/1602 objects degraded (15.169%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:55.935 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:55 smithi069 ceph-mon[63133]: osdmap e127: 8 total, 8 up, 8 in 2024-04-22T23:20:56.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:55 smithi149 ceph-mon[49797]: osdmap e126: 8 total, 8 up, 8 in 2024-04-22T23:20:56.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:55 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 243/1602 objects degraded (15.169%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T23:20:56.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:55 smithi149 ceph-mon[49797]: osdmap e127: 8 total, 8 up, 8 in 2024-04-22T23:20:56.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 253515 10.08 MB/sec execute 466 sec latency 241.057 ms 2024-04-22T23:20:57.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:57 smithi069 ceph-mon[63133]: pgmap v212: 97 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+degraded, 12 active+undersized, 23 active+undersized+degraded, 59 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s wr, 407 op/s; 220/1584 objects degraded (13.889%); 4.0 KiB/s, 1 objects/s recovering 2024-04-22T23:20:57.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:57 smithi069 ceph-mon[63133]: osdmap e128: 8 total, 8 up, 8 in 2024-04-22T23:20:57.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 253734 10.07 MB/sec execute 467 sec latency 1018.171 ms 2024-04-22T23:20:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:57 smithi149 ceph-mon[49797]: pgmap v212: 97 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+degraded, 12 active+undersized, 23 active+undersized+degraded, 59 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s wr, 407 op/s; 220/1584 objects degraded (13.889%); 4.0 KiB/s, 1 objects/s recovering 2024-04-22T23:20:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:57 smithi149 ceph-mon[49797]: osdmap e128: 8 total, 8 up, 8 in 2024-04-22T23:20:58.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 254018 10.07 MB/sec execute 468 sec latency 651.114 ms 2024-04-22T23:20:58.303 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:58 smithi069 ceph-mon[63133]: osdmap e129: 8 total, 8 up, 8 in 2024-04-22T23:20:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:58 smithi149 ceph-mon[49797]: osdmap e129: 8 total, 8 up, 8 in 2024-04-22T23:20:59.073 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:59 smithi069 ceph-mon[63133]: pgmap v215: 97 pgs: 2 peering, 2 active+recovery_wait+degraded, 1 active+recovering+degraded, 1 active+undersized+remapped, 91 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.1 MiB/s wr, 406 op/s; 25/1527 objects degraded (1.637%); 2.0 MiB/s, 53 keys/s, 61 objects/s recovering 2024-04-22T23:20:59.074 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:20:59 smithi069 ceph-mon[63133]: osdmap e130: 8 total, 8 up, 8 in 2024-04-22T23:20:59.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 254385 10.08 MB/sec execute 469 sec latency 44.262 ms 2024-04-22T23:20:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:59 smithi149 ceph-mon[49797]: pgmap v215: 97 pgs: 2 peering, 2 active+recovery_wait+degraded, 1 active+recovering+degraded, 1 active+undersized+remapped, 91 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.1 MiB/s wr, 406 op/s; 25/1527 objects degraded (1.637%); 2.0 MiB/s, 53 keys/s, 61 objects/s recovering 2024-04-22T23:20:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:20:59 smithi149 ceph-mon[49797]: osdmap e130: 8 total, 8 up, 8 in 2024-04-22T23:21:00.099 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:00 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 25/1527 objects degraded (1.637%), 3 pgs degraded (PG_DEGRADED) 2024-04-22T23:21:00.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 254769 10.08 MB/sec execute 470 sec latency 53.022 ms 2024-04-22T23:21:00.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:00 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 25/1527 objects degraded (1.637%), 3 pgs degraded (PG_DEGRADED) 2024-04-22T23:21:01.124 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:01 smithi069 ceph-mon[63133]: pgmap v217: 97 pgs: 2 peering, 2 active+recovery_wait+degraded, 1 active+recovering+degraded, 1 active+undersized+remapped, 91 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.2 MiB/s wr, 319 op/s; 25/1527 objects degraded (1.637%); 1.6 MiB/s, 44 keys/s, 50 objects/s recovering 2024-04-22T23:21:01.124 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:01.124 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:21:01.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 255095 10.07 MB/sec execute 471 sec latency 36.183 ms 2024-04-22T23:21:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:01 smithi149 ceph-mon[49797]: pgmap v217: 97 pgs: 2 peering, 2 active+recovery_wait+degraded, 1 active+recovering+degraded, 1 active+undersized+remapped, 91 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.2 MiB/s wr, 319 op/s; 25/1527 objects degraded (1.637%); 1.6 MiB/s, 44 keys/s, 50 objects/s recovering 2024-04-22T23:21:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:21:02.159 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:02 smithi069 ceph-mon[63133]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 25/1527 objects degraded (1.637%), 3 pgs degraded) 2024-04-22T23:21:02.160 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:02 smithi069 ceph-mon[63133]: Cluster is now healthy 2024-04-22T23:21:02.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 255417 10.07 MB/sec execute 472 sec latency 49.422 ms 2024-04-22T23:21:02.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:02 smithi149 ceph-mon[49797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 25/1527 objects degraded (1.637%), 3 pgs degraded) 2024-04-22T23:21:02.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:02 smithi149 ceph-mon[49797]: Cluster is now healthy 2024-04-22T23:21:03.190 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:03 smithi069 ceph-mon[63133]: pgmap v218: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.3 MiB/s wr, 532 op/s; 1.5 MiB/s, 35 keys/s, 48 objects/s recovering 2024-04-22T23:21:03.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 255703 10.07 MB/sec execute 473 sec latency 44.992 ms 2024-04-22T23:21:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:03 smithi149 ceph-mon[49797]: pgmap v218: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.3 MiB/s wr, 532 op/s; 1.5 MiB/s, 35 keys/s, 48 objects/s recovering 2024-04-22T23:21:04.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 255985 10.07 MB/sec execute 474 sec latency 56.065 ms 2024-04-22T23:21:04.963 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:21:05.237 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:05 smithi069 ceph-mon[63133]: pgmap v219: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 515 op/s; 1.2 MiB/s, 27 keys/s, 38 objects/s recovering 2024-04-22T23:21:05.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 256217 10.06 MB/sec execute 475 sec latency 52.522 ms 2024-04-22T23:21:05.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:05 smithi149 ceph-mon[49797]: pgmap v219: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 515 op/s; 1.2 MiB/s, 27 keys/s, 38 objects/s recovering 2024-04-22T23:21:05.958 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:21:05.958 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (7m) 60s ago 16m 23.2M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:21:05.958 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (4m) 60s ago 16m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:21:05.958 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (3m) 20s ago 15m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:21:05.958 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (7m) 60s ago 16m 57.1M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:21:05.958 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (10m) 60s ago 10m 38.4M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:21:05.958 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (10m) 20s ago 10m 15.3M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:21:05.958 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (6m) 60s ago 17m 509M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:21:05.958 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (5m) 20s ago 15m 452M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:21:05.959 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (5m) 60s ago 17m 51.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:21:05.959 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (5m) 20s ago 15m 40.2M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:21:05.959 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (10m) 60s ago 10m 264M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:21:05.959 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (10m) 20s ago 10m 86.8M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:21:05.959 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (7m) 60s ago 16m 17.0M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:21:05.959 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (6m) 20s ago 15m 19.7M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:21:05.959 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (3m) 60s ago 14m 345M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:21:05.959 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (2m) 60s ago 14m 283M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:21:05.959 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (104s) 60s ago 13m 210M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:21:05.959 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (71s) 60s ago 13m 12.7M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 e7772e3632e1 2024-04-22T23:21:05.960 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (25s) 20s ago 13m 11.3M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 036a1a8ba3c9 2024-04-22T23:21:05.960 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (12m) 20s ago 12m 589M 4096M 17.2.0 e1d6a67b021e 15430b00c430 2024-04-22T23:21:05.960 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (12m) 20s ago 12m 505M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:21:05.960 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (11m) 20s ago 11m 790M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:21:05.960 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (5m) 60s ago 15m 73.9M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:21:06.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 256471 10.05 MB/sec execute 476 sec latency 87.307 ms 2024-04-22T23:21:06.313 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:06 smithi069 ceph-mon[63133]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:06.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:06 smithi149 ceph-mon[49797]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:06.568 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:21:06.568 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:21:06.568 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:21:06.568 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:21:06.568 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:21:06.568 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:21:06.568 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:21:06.568 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:21:06.568 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-22T23:21:06.568 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 5 2024-04-22T23:21:06.568 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:21:06.569 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:21:06.569 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:21:06.569 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:21:06.569 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:21:06.569 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-22T23:21:06.569 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 9 2024-04-22T23:21:06.569 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:21:06.569 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:21:07.086 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:21:07.086 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:21:07.086 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:21:07.086 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:21:07.086 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:21:07.086 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:21:07.086 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:21:07.086 INFO:teuthology.orchestra.run.smithi069.stdout: "mon" 2024-04-22T23:21:07.086 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:21:07.086 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "11/23 daemons upgraded", 2024-04-22T23:21:07.086 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T23:21:07.086 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:21:07.087 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:21:07.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 256740 10.04 MB/sec execute 477 sec latency 56.224 ms 2024-04-22T23:21:07.327 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:07 smithi069 ceph-mon[63133]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:07.327 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:07 smithi069 ceph-mon[63133]: pgmap v220: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.6 MiB/s wr, 310 op/s; 185 KiB/s, 24 keys/s, 8 objects/s recovering 2024-04-22T23:21:07.328 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:07 smithi069 ceph-mon[63133]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:07.328 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:07 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T23:21:07.328 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:07 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T23:21:07.328 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:07 smithi069 ceph-mon[63133]: Upgrade: osd.5 is safe to restart 2024-04-22T23:21:07.328 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:07 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/1809518040' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:21:07.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:07 smithi149 ceph-mon[49797]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:07.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:07 smithi149 ceph-mon[49797]: pgmap v220: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.6 MiB/s wr, 310 op/s; 185 KiB/s, 24 keys/s, 8 objects/s recovering 2024-04-22T23:21:07.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:07 smithi149 ceph-mon[49797]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:07.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:07 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T23:21:07.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:07 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T23:21:07.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:07 smithi149 ceph-mon[49797]: Upgrade: osd.5 is safe to restart 2024-04-22T23:21:07.332 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:07 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/1809518040' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:21:07.643 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:21:08.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 256963 10.03 MB/sec execute 478 sec latency 56.305 ms 2024-04-22T23:21:08.365 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:08 smithi069 ceph-mon[63133]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:08.365 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:08 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/1528317133' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:21:08.430 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:08 smithi149 ceph-mon[49797]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:08.431 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:08 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/1528317133' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:21:09.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 257241 10.02 MB/sec execute 479 sec latency 65.356 ms 2024-04-22T23:21:09.390 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:09 smithi069 ceph-mon[63133]: pgmap v221: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 446 op/s; 150 KiB/s, 5 objects/s recovering 2024-04-22T23:21:09.391 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:09.391 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T23:21:09.391 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:21:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:09 smithi149 ceph-mon[49797]: pgmap v221: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 446 op/s; 150 KiB/s, 5 objects/s recovering 2024-04-22T23:21:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T23:21:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:21:10.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 257551 10.02 MB/sec execute 480 sec latency 58.634 ms 2024-04-22T23:21:10.415 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:10 smithi069 ceph-mon[63133]: Upgrade: Updating osd.5 2024-04-22T23:21:10.415 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:10 smithi069 ceph-mon[63133]: Deploying daemon osd.5 on smithi149 2024-04-22T23:21:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:10 smithi149 ceph-mon[49797]: Upgrade: Updating osd.5 2024-04-22T23:21:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:10 smithi149 ceph-mon[49797]: Deploying daemon osd.5 on smithi149 2024-04-22T23:21:11.198 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:11 smithi069 ceph-mon[63133]: pgmap v222: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 381 op/s; 128 KiB/s, 4 objects/s recovering 2024-04-22T23:21:11.199 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:11 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:11.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 257823 10.02 MB/sec execute 481 sec latency 76.281 ms 2024-04-22T23:21:11.396 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:11 smithi149 ceph-mon[49797]: pgmap v222: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 381 op/s; 128 KiB/s, 4 objects/s recovering 2024-04-22T23:21:11.396 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:11 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:12.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 258230 10.02 MB/sec execute 482 sec latency 100.139 ms 2024-04-22T23:21:12.556 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:12 smithi149 systemd[1]: Stopping Ceph osd.5 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:21:12.950 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:12 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[31155]: 2024-04-22T23:21:12.631+0000 7fd5ba200700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:21:12.950 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:12 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[31155]: 2024-04-22T23:21:12.631+0000 7fd5ba200700 -1 osd.5 130 *** Got signal Terminated *** 2024-04-22T23:21:12.950 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:12 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[31155]: 2024-04-22T23:21:12.631+0000 7fd5ba200700 -1 osd.5 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:21:13.176 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:13 smithi069 ceph-mon[63133]: pgmap v223: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 520 op/s; 125 KiB/s, 4 objects/s recovering 2024-04-22T23:21:13.177 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:13 smithi069 ceph-mon[63133]: osd.5 marked itself down and dead 2024-04-22T23:21:13.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 258604 10.02 MB/sec execute 483 sec latency 119.127 ms 2024-04-22T23:21:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:13 smithi149 ceph-mon[49797]: pgmap v223: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 520 op/s; 125 KiB/s, 4 objects/s recovering 2024-04-22T23:21:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:13 smithi149 ceph-mon[49797]: osd.5 marked itself down and dead 2024-04-22T23:21:14.099 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:13 smithi149 podman[56540]: 2024-04-22 23:21:13.811969288 +0000 UTC m=+1.318606058 container died 15430b00c4301e2485c2004d9d1e0e658c4808c92135979ff65b401a38885870 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, name=centos-stream, version=8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vcs-type=git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-22T23:21:14.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 258727 10.00 MB/sec execute 484 sec latency 1022.109 ms 2024-04-22T23:21:14.409 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:14 smithi069 ceph-mon[63133]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:21:14.410 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:14 smithi069 ceph-mon[63133]: osdmap e131: 8 total, 7 up, 8 in 2024-04-22T23:21:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:14 smithi149 ceph-mon[49797]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:21:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:14 smithi149 ceph-mon[49797]: osdmap e131: 8 total, 7 up, 8 in 2024-04-22T23:21:14.450 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:14 smithi149 podman[56540]: 2024-04-22 23:21:14.306535298 +0000 UTC m=+1.813172122 container cleanup 15430b00c4301e2485c2004d9d1e0e658c4808c92135979ff65b401a38885870 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vcs-type=git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, ceph=True, architecture=x86_64, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream) 2024-04-22T23:21:14.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:14 smithi149 bash[56540]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5 2024-04-22T23:21:14.950 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:14 smithi149 podman[56557]: 2024-04-22 23:21:14.641817169 +0000 UTC m=+0.825207408 container remove 15430b00c4301e2485c2004d9d1e0e658c4808c92135979ff65b401a38885870 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, name=centos-stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870) 2024-04-22T23:21:15.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:15 smithi069 ceph-mon[63133]: pgmap v225: 97 pgs: 12 stale+active+clean, 85 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 396 op/s 2024-04-22T23:21:15.176 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:15 smithi069 ceph-mon[63133]: osdmap e132: 8 total, 7 up, 8 in 2024-04-22T23:21:15.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 259101 10.00 MB/sec execute 485 sec latency 40.075 ms 2024-04-22T23:21:15.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:15 smithi149 ceph-mon[49797]: pgmap v225: 97 pgs: 12 stale+active+clean, 85 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 396 op/s 2024-04-22T23:21:15.329 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:15 smithi149 ceph-mon[49797]: osdmap e132: 8 total, 7 up, 8 in 2024-04-22T23:21:15.330 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:15 smithi149 podman[56640]: 2024-04-22 23:21:14.925906526 +0000 UTC m=+0.023413041 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:21:15.330 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:15 smithi149 podman[56640]: 2024-04-22 23:21:15.067772048 +0000 UTC m=+0.165278545 container create 3adbfbf57baad59dd79548ffaa3026c0877189db24cffea9ffd1fa0d18feb782 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-deactivate, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-22T23:21:15.700 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:15 smithi149 podman[56640]: 2024-04-22 23:21:15.329382544 +0000 UTC m=+0.426889061 container init 3adbfbf57baad59dd79548ffaa3026c0877189db24cffea9ffd1fa0d18feb782 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux ) 2024-04-22T23:21:15.700 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:15 smithi149 podman[56640]: 2024-04-22 23:21:15.333973181 +0000 UTC m=+0.431479679 container start 3adbfbf57baad59dd79548ffaa3026c0877189db24cffea9ffd1fa0d18feb782 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-deactivate, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD) 2024-04-22T23:21:15.700 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:15 smithi149 podman[56640]: 2024-04-22 23:21:15.437713916 +0000 UTC m=+0.535220414 container attach 3adbfbf57baad59dd79548ffaa3026c0877189db24cffea9ffd1fa0d18feb782 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416) 2024-04-22T23:21:16.150 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:15 smithi149 podman[56640]: 2024-04-22 23:21:15.782842097 +0000 UTC m=+0.880348603 container died 3adbfbf57baad59dd79548ffaa3026c0877189db24cffea9ffd1fa0d18feb782 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True) 2024-04-22T23:21:16.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 259549 10.02 MB/sec execute 486 sec latency 46.371 ms 2024-04-22T23:21:16.862 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:16 smithi149 podman[56666]: 2024-04-22 23:21:16.577028413 +0000 UTC m=+0.793449240 container remove 3adbfbf57baad59dd79548ffaa3026c0877189db24cffea9ffd1fa0d18feb782 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-deactivate, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240416) 2024-04-22T23:21:16.863 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:16 smithi149 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.5.service: Deactivated successfully. 2024-04-22T23:21:16.863 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:16 smithi149 systemd[1]: Stopped Ceph osd.5 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:21:16.863 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:16 smithi149 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.5.service: Consumed 5min 23.590s CPU time. 2024-04-22T23:21:16.863 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:16 smithi149 ceph-mon[49797]: pgmap v227: 97 pgs: 1 active+undersized, 2 active+undersized+degraded, 12 stale+active+clean, 82 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.5 MiB/s wr, 383 op/s; 25/1593 objects degraded (1.569%) 2024-04-22T23:21:16.863 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:16.863 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:21:16.864 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:16 smithi149 ceph-mon[49797]: Health check failed: Degraded data redundancy: 25/1593 objects degraded (1.569%), 2 pgs degraded (PG_DEGRADED) 2024-04-22T23:21:16.977 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:16 smithi069 ceph-mon[63133]: pgmap v227: 97 pgs: 1 active+undersized, 2 active+undersized+degraded, 12 stale+active+clean, 82 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.5 MiB/s wr, 383 op/s; 25/1593 objects degraded (1.569%) 2024-04-22T23:21:16.977 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:16.977 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:21:16.977 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:16 smithi069 ceph-mon[63133]: Health check failed: Degraded data redundancy: 25/1593 objects degraded (1.569%), 2 pgs degraded (PG_DEGRADED) 2024-04-22T23:21:17.170 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:16 smithi149 systemd[1]: Starting Ceph osd.5 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:21:17.170 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:17 smithi149 podman[56763]: 2024-04-22 23:21:17.01214149 +0000 UTC m=+0.024581427 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:21:17.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 260048 10.02 MB/sec execute 487 sec latency 34.755 ms 2024-04-22T23:21:17.450 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:17 smithi149 podman[56763]: 2024-04-22 23:21:17.170232679 +0000 UTC m=+0.182672664 container create 2872a34d945900516dfc23c8ac224f48d8efee8b6ae5401f2f726a448402a813 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T23:21:17.950 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:17 smithi149 podman[56763]: 2024-04-22 23:21:17.481684358 +0000 UTC m=+0.494124296 container init 2872a34d945900516dfc23c8ac224f48d8efee8b6ae5401f2f726a448402a813 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T23:21:17.950 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:17 smithi149 podman[56763]: 2024-04-22 23:21:17.487295984 +0000 UTC m=+0.499735926 container start 2872a34d945900516dfc23c8ac224f48d8efee8b6ae5401f2f726a448402a813 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-22T23:21:17.950 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:17 smithi149 podman[56763]: 2024-04-22 23:21:17.598220106 +0000 UTC m=+0.610660047 container attach 2872a34d945900516dfc23c8ac224f48d8efee8b6ae5401f2f726a448402a813 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-22T23:21:18.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 260353 10.01 MB/sec execute 488 sec latency 57.914 ms 2024-04-22T23:21:18.450 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate[56778]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate[56778]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 bash[56763]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 bash[56763]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 bash[56763]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 bash[56763]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 bash[56763]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 bash[56763]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 bash[56763]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 bash[56763]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 bash[56763]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate[56778]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate[56778]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T23:21:18.451 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate[56778]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T23:21:18.452 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate[56778]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T23:21:18.452 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate[56778]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-22T23:21:18.452 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate[56778]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T23:21:18.452 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate[56778]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-22T23:21:18.452 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 podman[56763]: 2024-04-22 23:21:18.178676156 +0000 UTC m=+1.191116084 container died 2872a34d945900516dfc23c8ac224f48d8efee8b6ae5401f2f726a448402a813 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-22T23:21:18.887 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:18 smithi149 ceph-mon[49797]: pgmap v228: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.5 MiB/s wr, 538 op/s; 174/1617 objects degraded (10.761%) 2024-04-22T23:21:19.038 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:18 smithi069 ceph-mon[63133]: pgmap v228: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.5 MiB/s wr, 538 op/s; 174/1617 objects degraded (10.761%) 2024-04-22T23:21:19.138 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:18 smithi149 podman[56911]: 2024-04-22 23:21:18.886942933 +0000 UTC m=+0.701339953 container remove 2872a34d945900516dfc23c8ac224f48d8efee8b6ae5401f2f726a448402a813 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2) 2024-04-22T23:21:19.138 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:19 smithi149 podman[56974]: 2024-04-22 23:21:19.017519608 +0000 UTC m=+0.022859815 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:21:19.138 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:19 smithi149 podman[56974]: 2024-04-22 23:21:19.13797505 +0000 UTC m=+0.143315229 container create 539c90bd7194849249e61c0c7e3929a7c7e62944c59e34b8d3201eda9b4817eb (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-22T23:21:19.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 260733 10.01 MB/sec execute 489 sec latency 45.776 ms 2024-04-22T23:21:19.700 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:19 smithi149 podman[56974]: 2024-04-22 23:21:19.432502712 +0000 UTC m=+0.437842891 container init 539c90bd7194849249e61c0c7e3929a7c7e62944c59e34b8d3201eda9b4817eb (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T23:21:19.701 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:19 smithi149 podman[56974]: 2024-04-22 23:21:19.435770959 +0000 UTC m=+0.441111141 container start 539c90bd7194849249e61c0c7e3929a7c7e62944c59e34b8d3201eda9b4817eb (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T23:21:19.701 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:19 smithi149 ceph-osd[57025]: -- 172.21.15.149:0/444952027 <== mon.1 v2:172.21.15.149:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c3a723f680 con 0x55c3a7208c00 2024-04-22T23:21:19.701 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:19 smithi149 bash[56974]: 539c90bd7194849249e61c0c7e3929a7c7e62944c59e34b8d3201eda9b4817eb 2024-04-22T23:21:19.701 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:19 smithi149 systemd[1]: Started Ceph osd.5 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:21:20.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 261081 10.01 MB/sec execute 490 sec latency 49.078 ms 2024-04-22T23:21:20.450 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:20 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:21:20.004+0000 7f61de2277c0 -1 Falling back to public interface 2024-04-22T23:21:20.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:20.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:20.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:21:20.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:20 smithi069 ceph-mon[63133]: pgmap v229: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.4 MiB/s wr, 315 op/s; 174/1617 objects degraded (10.761%) 2024-04-22T23:21:20.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:20.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:20.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:21:20.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:20 smithi149 ceph-mon[49797]: pgmap v229: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.4 MiB/s wr, 315 op/s; 174/1617 objects degraded (10.761%) 2024-04-22T23:21:21.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 261415 10.01 MB/sec execute 491 sec latency 53.241 ms 2024-04-22T23:21:21.890 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:21 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 187/1647 objects degraded (11.354%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T23:21:21.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:21 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 187/1647 objects degraded (11.354%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T23:21:22.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 261767 10.01 MB/sec execute 492 sec latency 45.076 ms 2024-04-22T23:21:22.911 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:22 smithi069 ceph-mon[63133]: pgmap v230: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 457 op/s; 187/1647 objects degraded (11.354%) 2024-04-22T23:21:23.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:22 smithi149 ceph-mon[49797]: pgmap v230: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 457 op/s; 187/1647 objects degraded (11.354%) 2024-04-22T23:21:23.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 262050 10.01 MB/sec execute 493 sec latency 52.206 ms 2024-04-22T23:21:24.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 262471 10.01 MB/sec execute 494 sec latency 36.437 ms 2024-04-22T23:21:25.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 262703 10.00 MB/sec execute 495 sec latency 41.451 ms 2024-04-22T23:21:25.462 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:25 smithi069 ceph-mon[63133]: pgmap v231: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.3 MiB/s wr, 544 op/s; 191/1659 objects degraded (11.513%) 2024-04-22T23:21:25.462 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:25 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:25.462 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:25 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:25 smithi149 ceph-mon[49797]: pgmap v231: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.3 MiB/s wr, 544 op/s; 191/1659 objects degraded (11.513%) 2024-04-22T23:21:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:25 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:25 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:26.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 262978 10.00 MB/sec execute 496 sec latency 64.798 ms 2024-04-22T23:21:26.681 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:26 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:26.682 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:26 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:26.682 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:26 smithi149 ceph-mon[49797]: pgmap v232: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.0 MiB/s wr, 484 op/s; 194/1659 objects degraded (11.694%) 2024-04-22T23:21:26.740 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:26 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:26.741 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:26 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:26.741 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:26 smithi069 ceph-mon[63133]: pgmap v232: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.0 MiB/s wr, 484 op/s; 194/1659 objects degraded (11.694%) 2024-04-22T23:21:27.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 263271 10.00 MB/sec execute 497 sec latency 69.350 ms 2024-04-22T23:21:27.901 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:27 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 180/1632 objects degraded (11.029%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T23:21:28.116 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:27 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 180/1632 objects degraded (11.029%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T23:21:28.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 263568 9.99 MB/sec execute 498 sec latency 65.705 ms 2024-04-22T23:21:28.445 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:28 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:21:28.229+0000 7f61de2277c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-22T23:21:29.048 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:28 smithi069 ceph-mon[63133]: pgmap v233: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.7 MiB/s wr, 548 op/s; 180/1632 objects degraded (11.029%) 2024-04-22T23:21:29.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:28 smithi149 ceph-mon[49797]: pgmap v233: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.7 MiB/s wr, 548 op/s; 180/1632 objects degraded (11.029%) 2024-04-22T23:21:29.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 263839 9.98 MB/sec execute 499 sec latency 46.511 ms 2024-04-22T23:21:30.078 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:29 smithi069 ceph-mon[63133]: from='osd.5 [v2:172.21.15.149:6808/2457137087,v1:172.21.15.149:6809/2457137087]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T23:21:30.078 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:29 smithi069 ceph-mon[63133]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T23:21:30.078 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:30.078 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:30.078 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:21:30.078 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:21:30.078 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:29 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:30.200 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:29 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:21:29.813+0000 7f61de2277c0 -1 osd.5 130 log_to_monitors true 2024-04-22T23:21:30.200 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:21:29 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:21:29.946+0000 7f61cf67f700 -1 osd.5 130 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:21:30.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:29 smithi149 ceph-mon[49797]: from='osd.5 [v2:172.21.15.149:6808/2457137087,v1:172.21.15.149:6809/2457137087]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T23:21:30.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:29 smithi149 ceph-mon[49797]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T23:21:30.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:30.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:30.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:21:30.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:21:30.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:29 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:30.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 264076 9.97 MB/sec execute 500 sec latency 40.943 ms 2024-04-22T23:21:31.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: pgmap v234: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 445 op/s; 180/1632 objects degraded (11.029%) 2024-04-22T23:21:31.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T23:21:31.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: osdmap e133: 8 total, 7 up, 8 in 2024-04-22T23:21:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: from='osd.5 [v2:172.21.15.149:6808/2457137087,v1:172.21.15.149:6809/2457137087]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:21:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:21:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:21:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:21:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:21:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:21:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T23:21:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T23:21:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-22T23:21:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:21:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: pgmap v234: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 445 op/s; 180/1632 objects degraded (11.029%) 2024-04-22T23:21:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T23:21:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: osdmap e133: 8 total, 7 up, 8 in 2024-04-22T23:21:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: from='osd.5 [v2:172.21.15.149:6808/2457137087,v1:172.21.15.149:6809/2457137087]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:21:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:21:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:21:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:21:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:21:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:21:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T23:21:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T23:21:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-22T23:21:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:31.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:21:31.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 264294 9.97 MB/sec execute 501 sec latency 382.145 ms 2024-04-22T23:21:32.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:31 smithi149 ceph-mon[49797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:21:32.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:31 smithi149 ceph-mon[49797]: osd.5 [v2:172.21.15.149:6808/2457137087,v1:172.21.15.149:6809/2457137087] boot 2024-04-22T23:21:32.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:31 smithi149 ceph-mon[49797]: osdmap e134: 8 total, 8 up, 8 in 2024-04-22T23:21:32.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:21:32.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:31 smithi149 ceph-mon[49797]: osdmap e135: 8 total, 8 up, 8 in 2024-04-22T23:21:32.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 264294 9.95 MB/sec execute 502 sec latency 1382.200 ms 2024-04-22T23:21:32.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:31 smithi069 ceph-mon[63133]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:21:32.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:31 smithi069 ceph-mon[63133]: osd.5 [v2:172.21.15.149:6808/2457137087,v1:172.21.15.149:6809/2457137087] boot 2024-04-22T23:21:32.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:31 smithi069 ceph-mon[63133]: osdmap e134: 8 total, 8 up, 8 in 2024-04-22T23:21:32.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T23:21:32.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:31 smithi069 ceph-mon[63133]: osdmap e135: 8 total, 8 up, 8 in 2024-04-22T23:21:33.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 264294 9.93 MB/sec execute 503 sec latency 2382.228 ms 2024-04-22T23:21:33.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:32 smithi069 ceph-mon[63133]: pgmap v237: 97 pgs: 8 peering, 5 active+undersized, 18 active+undersized+degraded, 66 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.9 MiB/s wr, 327 op/s; 139/1596 objects degraded (8.709%) 2024-04-22T23:21:33.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:32 smithi069 ceph-mon[63133]: osdmap e136: 8 total, 8 up, 8 in 2024-04-22T23:21:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:32 smithi149 ceph-mon[49797]: pgmap v237: 97 pgs: 8 peering, 5 active+undersized, 18 active+undersized+degraded, 66 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.9 MiB/s wr, 327 op/s; 139/1596 objects degraded (8.709%) 2024-04-22T23:21:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:32 smithi149 ceph-mon[49797]: osdmap e136: 8 total, 8 up, 8 in 2024-04-22T23:21:34.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 264447 9.91 MB/sec execute 504 sec latency 2487.526 ms 2024-04-22T23:21:34.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:34 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 97/1602 objects degraded (6.055%), 9 pgs degraded (PG_DEGRADED) 2024-04-22T23:21:34.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:34 smithi069 ceph-mon[63133]: osdmap e137: 8 total, 8 up, 8 in 2024-04-22T23:21:34.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:34 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 97/1602 objects degraded (6.055%), 9 pgs degraded (PG_DEGRADED) 2024-04-22T23:21:34.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:34 smithi149 ceph-mon[49797]: osdmap e137: 8 total, 8 up, 8 in 2024-04-22T23:21:35.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:35 smithi069 ceph-mon[63133]: pgmap v240: 97 pgs: 1 active+recovery_wait+degraded, 4 remapped+peering, 11 peering, 3 active+undersized, 8 active+undersized+degraded, 70 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.0 MiB/s wr, 423 op/s; 97/1602 objects degraded (6.055%); 22 B/s, 0 objects/s recovering 2024-04-22T23:21:35.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:35 smithi069 ceph-mon[63133]: osdmap e138: 8 total, 8 up, 8 in 2024-04-22T23:21:35.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 264580 9.90 MB/sec execute 505 sec latency 1038.631 ms 2024-04-22T23:21:35.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:35 smithi149 ceph-mon[49797]: pgmap v240: 97 pgs: 1 active+recovery_wait+degraded, 4 remapped+peering, 11 peering, 3 active+undersized, 8 active+undersized+degraded, 70 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.0 MiB/s wr, 423 op/s; 97/1602 objects degraded (6.055%); 22 B/s, 0 objects/s recovering 2024-04-22T23:21:35.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:35 smithi149 ceph-mon[49797]: osdmap e138: 8 total, 8 up, 8 in 2024-04-22T23:21:36.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 264898 9.90 MB/sec execute 506 sec latency 733.008 ms 2024-04-22T23:21:37.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:36 smithi069 ceph-mon[63133]: pgmap v243: 97 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 14 peering, 1 active+undersized+remapped, 80 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s wr, 400 op/s; 22/1581 objects degraded (1.392%); 1.2 MiB/s, 31 objects/s recovering 2024-04-22T23:21:37.014 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:36 smithi069 ceph-mon[63133]: osdmap e139: 8 total, 8 up, 8 in 2024-04-22T23:21:37.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 265294 9.90 MB/sec execute 507 sec latency 34.512 ms 2024-04-22T23:21:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:36 smithi149 ceph-mon[49797]: pgmap v243: 97 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 14 peering, 1 active+undersized+remapped, 80 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s wr, 400 op/s; 22/1581 objects degraded (1.392%); 1.2 MiB/s, 31 objects/s recovering 2024-04-22T23:21:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:36 smithi149 ceph-mon[49797]: osdmap e139: 8 total, 8 up, 8 in 2024-04-22T23:21:38.033 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:38 smithi069 ceph-mon[63133]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/1581 objects degraded (1.392%), 1 pg degraded) 2024-04-22T23:21:38.033 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:38 smithi069 ceph-mon[63133]: Cluster is now healthy 2024-04-22T23:21:38.189 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:21:38.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 265632 9.89 MB/sec execute 508 sec latency 37.778 ms 2024-04-22T23:21:38.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:38 smithi149 ceph-mon[49797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/1581 objects degraded (1.392%), 1 pg degraded) 2024-04-22T23:21:38.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:38 smithi149 ceph-mon[49797]: Cluster is now healthy 2024-04-22T23:21:39.047 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:39 smithi069 ceph-mon[63133]: pgmap v245: 97 pgs: 1 remapped+peering, 3 peering, 1 active+undersized+remapped, 92 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.0 MiB/s wr, 176 op/s; 2.0 MiB/s, 43 objects/s recovering 2024-04-22T23:21:39.047 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:39 smithi069 ceph-mon[63133]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:39.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 265957 9.89 MB/sec execute 509 sec latency 46.016 ms 2024-04-22T23:21:39.306 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:21:39.306 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (7m) 94s ago 16m 23.2M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:21:39.306 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (4m) 94s ago 16m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:21:39.306 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (4m) 15s ago 16m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:21:39.306 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (7m) 94s ago 16m 57.1M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:21:39.306 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (11m) 94s ago 11m 38.4M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:21:39.306 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (11m) 15s ago 11m 15.4M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:21:39.306 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (7m) 94s ago 18m 509M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:21:39.306 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (6m) 15s ago 16m 452M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (6m) 94s ago 18m 51.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (5m) 15s ago 16m 43.3M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (10m) 94s ago 10m 264M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (10m) 15s ago 10m 86.9M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (7m) 94s ago 16m 17.0M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (7m) 15s ago 16m 19.8M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (3m) 94s ago 15m 345M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (3m) 94s ago 14m 283M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (2m) 94s ago 14m 210M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (104s) 94s ago 14m 12.7M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 e7772e3632e1 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (58s) 15s ago 13m 286M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 036a1a8ba3c9 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (20s) 15s ago 13m 12.5M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 539c90bd7194 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (12m) 15s ago 12m 536M 4096M 17.2.0 e1d6a67b021e 42b8779eb68d 2024-04-22T23:21:39.307 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (12m) 15s ago 12m 559M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:21:39.308 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (6m) 94s ago 15m 73.9M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:21:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:39 smithi149 ceph-mon[49797]: pgmap v245: 97 pgs: 1 remapped+peering, 3 peering, 1 active+undersized+remapped, 92 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.0 MiB/s wr, 176 op/s; 2.0 MiB/s, 43 objects/s recovering 2024-04-22T23:21:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:39 smithi149 ceph-mon[49797]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:39.944 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:21:39.944 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:21:39.944 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:21:39.944 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 6 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-22T23:21:39.945 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 10 2024-04-22T23:21:39.946 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:21:39.946 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:21:40.074 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:40 smithi069 ceph-mon[63133]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:40.074 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:40 smithi069 ceph-mon[63133]: from='client.44295 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:40.074 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:40 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/2485773613' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:21:40.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 266219 9.89 MB/sec execute 510 sec latency 41.896 ms 2024-04-22T23:21:40.442 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:21:40.442 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:21:40.442 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:21:40.442 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:21:40.443 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:21:40.443 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:21:40.443 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:21:40.443 INFO:teuthology.orchestra.run.smithi069.stdout: "mon" 2024-04-22T23:21:40.443 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:21:40.443 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "12/23 daemons upgraded", 2024-04-22T23:21:40.443 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T23:21:40.443 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:21:40.443 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:21:40.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:40 smithi149 ceph-mon[49797]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:40.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:40 smithi149 ceph-mon[49797]: from='client.44295 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:40.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:40 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/2485773613' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:21:41.088 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:41 smithi069 ceph-mon[63133]: pgmap v246: 97 pgs: 1 remapped+peering, 3 peering, 1 active+undersized+remapped, 92 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.7 MiB/s wr, 143 op/s; 1.6 MiB/s, 34 objects/s recovering 2024-04-22T23:21:41.169 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:21:41.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 266493 9.90 MB/sec execute 511 sec latency 51.533 ms 2024-04-22T23:21:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:41 smithi149 ceph-mon[49797]: pgmap v246: 97 pgs: 1 remapped+peering, 3 peering, 1 active+undersized+remapped, 92 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.7 MiB/s wr, 143 op/s; 1.6 MiB/s, 34 objects/s recovering 2024-04-22T23:21:42.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:42 smithi069 ceph-mon[63133]: from='client.44301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:42.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:42 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/4066829042' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:21:42.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 266807 9.89 MB/sec execute 512 sec latency 68.803 ms 2024-04-22T23:21:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:42 smithi149 ceph-mon[49797]: from='client.44301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:21:42.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:42 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/4066829042' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:21:43.136 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:43 smithi069 ceph-mon[63133]: pgmap v247: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.5 MiB/s wr, 305 op/s; 2.3 MiB/s, 30 objects/s recovering 2024-04-22T23:21:43.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 267141 9.88 MB/sec execute 513 sec latency 73.078 ms 2024-04-22T23:21:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:43 smithi149 ceph-mon[49797]: pgmap v247: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.5 MiB/s wr, 305 op/s; 2.3 MiB/s, 30 objects/s recovering 2024-04-22T23:21:44.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 267406 9.87 MB/sec execute 514 sec latency 40.680 ms 2024-04-22T23:21:45.206 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:45 smithi069 ceph-mon[63133]: pgmap v248: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 381 op/s; 1.5 MiB/s, 12 objects/s recovering 2024-04-22T23:21:45.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 267627 9.87 MB/sec execute 515 sec latency 68.137 ms 2024-04-22T23:21:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:45 smithi149 ceph-mon[49797]: pgmap v248: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 381 op/s; 1.5 MiB/s, 12 objects/s recovering 2024-04-22T23:21:46.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T23:21:46.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:46 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T23:21:46.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:46 smithi069 ceph-mon[63133]: Upgrade: osd.6 is safe to restart 2024-04-22T23:21:46.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:46.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:21:46.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 267908 9.86 MB/sec execute 516 sec latency 84.882 ms 2024-04-22T23:21:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T23:21:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:46 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T23:21:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:46 smithi149 ceph-mon[49797]: Upgrade: osd.6 is safe to restart 2024-04-22T23:21:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:21:47.281 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:47 smithi069 ceph-mon[63133]: pgmap v249: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 359 op/s; 1.3 MiB/s, 11 objects/s recovering 2024-04-22T23:21:47.281 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:47 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:47.281 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:47 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T23:21:47.281 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:47 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:21:47.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 268204 9.86 MB/sec execute 517 sec latency 44.254 ms 2024-04-22T23:21:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:47 smithi149 ceph-mon[49797]: pgmap v249: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 359 op/s; 1.3 MiB/s, 11 objects/s recovering 2024-04-22T23:21:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:47 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:47 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T23:21:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:47 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:21:48.154 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:48 smithi149 ceph-mon[49797]: Upgrade: Updating osd.6 2024-04-22T23:21:48.155 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:48 smithi149 ceph-mon[49797]: Deploying daemon osd.6 on smithi149 2024-04-22T23:21:48.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 268499 9.86 MB/sec execute 518 sec latency 66.596 ms 2024-04-22T23:21:48.302 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:48 smithi069 ceph-mon[63133]: Upgrade: Updating osd.6 2024-04-22T23:21:48.302 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:48 smithi069 ceph-mon[63133]: Deploying daemon osd.6 on smithi149 2024-04-22T23:21:49.169 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:49 smithi149 ceph-mon[49797]: pgmap v250: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.4 MiB/s wr, 461 op/s; 1.1 MiB/s, 9 objects/s recovering 2024-04-22T23:21:49.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 268822 9.86 MB/sec execute 519 sec latency 37.385 ms 2024-04-22T23:21:49.319 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:49 smithi069 ceph-mon[63133]: pgmap v250: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.4 MiB/s wr, 461 op/s; 1.1 MiB/s, 9 objects/s recovering 2024-04-22T23:21:50.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 269141 9.85 MB/sec execute 520 sec latency 61.947 ms 2024-04-22T23:21:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:51 smithi069 ceph-mon[63133]: pgmap v251: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 407 op/s; 686 KiB/s, 2 objects/s recovering 2024-04-22T23:21:51.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:51 smithi149 ceph-mon[49797]: pgmap v251: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 407 op/s; 686 KiB/s, 2 objects/s recovering 2024-04-22T23:21:51.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:51.201 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:51 smithi149 systemd[1]: Stopping Ceph osd.6 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:21:51.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 269410 9.85 MB/sec execute 521 sec latency 60.086 ms 2024-04-22T23:21:51.700 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:51 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[34194]: 2024-04-22T23:21:51.367+0000 7fb23c460700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:21:51.700 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:51 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[34194]: 2024-04-22T23:21:51.367+0000 7fb23c460700 -1 osd.6 139 *** Got signal Terminated *** 2024-04-22T23:21:51.700 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:51 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[34194]: 2024-04-22T23:21:51.367+0000 7fb23c460700 -1 osd.6 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:21:52.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:52 smithi069 ceph-mon[63133]: osd.6 marked itself down and dead 2024-04-22T23:21:52.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 269662 9.84 MB/sec execute 522 sec latency 205.216 ms 2024-04-22T23:21:52.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:52 smithi149 ceph-mon[49797]: osd.6 marked itself down and dead 2024-04-22T23:21:52.935 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:52 smithi149 podman[58804]: 2024-04-22 23:21:52.641377898 +0000 UTC m=+1.411396781 container died 42b8779eb68dc8d8c68aa8ba5b0908c74eef2c2153b5be3ac289685bb85ef147 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6, GIT_CLEAN=True, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, release=754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream) 2024-04-22T23:21:53.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:53 smithi149 ceph-mon[49797]: pgmap v252: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 456 op/s; 686 KiB/s, 2 objects/s recovering 2024-04-22T23:21:53.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:53 smithi149 ceph-mon[49797]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:21:53.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:53 smithi149 ceph-mon[49797]: osdmap e140: 8 total, 7 up, 8 in 2024-04-22T23:21:53.201 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:53 smithi149 podman[58804]: 2024-04-22 23:21:53.125395697 +0000 UTC m=+1.895414582 container cleanup 42b8779eb68dc8d8c68aa8ba5b0908c74eef2c2153b5be3ac289685bb85ef147 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6, io.openshift.expose-services=, distribution-scope=public, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, version=8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , RELEASE=HEAD, ceph=True, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-22T23:21:53.201 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:53 smithi149 bash[58804]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6 2024-04-22T23:21:53.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 269693 9.82 MB/sec execute 523 sec latency 1050.365 ms 2024-04-22T23:21:53.335 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:53 smithi069 ceph-mon[63133]: pgmap v252: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 456 op/s; 686 KiB/s, 2 objects/s recovering 2024-04-22T23:21:53.335 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:53 smithi069 ceph-mon[63133]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:21:53.335 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:53 smithi069 ceph-mon[63133]: osdmap e140: 8 total, 7 up, 8 in 2024-04-22T23:21:53.797 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:53 smithi149 podman[58822]: 2024-04-22 23:21:53.452163807 +0000 UTC m=+0.800664766 container remove 42b8779eb68dc8d8c68aa8ba5b0908c74eef2c2153b5be3ac289685bb85ef147 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6, io.openshift.expose-services=, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, distribution-scope=public, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream) 2024-04-22T23:21:54.073 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:53 smithi149 podman[58906]: 2024-04-22 23:21:53.699425619 +0000 UTC m=+0.022992162 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:21:54.074 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:53 smithi149 podman[58906]: 2024-04-22 23:21:53.811411293 +0000 UTC m=+0.134977821 container create b221d725433a39f2cfdc3be4d9cb0740ed41b2dafa7c1c0fdc7b6aebf2a19cc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-22T23:21:54.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:54 smithi069 ceph-mon[63133]: osdmap e141: 8 total, 7 up, 8 in 2024-04-22T23:21:54.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 270022 9.83 MB/sec execute 524 sec latency 41.758 ms 2024-04-22T23:21:54.436 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:54 smithi149 ceph-mon[49797]: osdmap e141: 8 total, 7 up, 8 in 2024-04-22T23:21:54.437 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:54 smithi149 podman[58906]: 2024-04-22 23:21:54.073153884 +0000 UTC m=+0.396720427 container init b221d725433a39f2cfdc3be4d9cb0740ed41b2dafa7c1c0fdc7b6aebf2a19cc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-22T23:21:54.437 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:54 smithi149 podman[58906]: 2024-04-22 23:21:54.077726638 +0000 UTC m=+0.401293170 container start b221d725433a39f2cfdc3be4d9cb0740ed41b2dafa7c1c0fdc7b6aebf2a19cc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-22T23:21:54.437 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:54 smithi149 podman[58906]: 2024-04-22 23:21:54.172882922 +0000 UTC m=+0.496449459 container attach b221d725433a39f2cfdc3be4d9cb0740ed41b2dafa7c1c0fdc7b6aebf2a19cc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-22T23:21:54.700 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:54 smithi149 podman[58931]: 2024-04-22 23:21:54.470143698 +0000 UTC m=+0.023671595 container died b221d725433a39f2cfdc3be4d9cb0740ed41b2dafa7c1c0fdc7b6aebf2a19cc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-22T23:21:55.136 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:55 smithi069 ceph-mon[63133]: pgmap v255: 97 pgs: 4 peering, 9 stale+active+clean, 84 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.5 MiB/s wr, 490 op/s 2024-04-22T23:21:55.137 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:55 smithi069 ceph-mon[63133]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T23:21:55.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 270332 9.82 MB/sec execute 525 sec latency 44.711 ms 2024-04-22T23:21:55.442 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:55 smithi149 ceph-mon[49797]: pgmap v255: 97 pgs: 4 peering, 9 stale+active+clean, 84 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.5 MiB/s wr, 490 op/s 2024-04-22T23:21:55.443 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:55 smithi149 ceph-mon[49797]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T23:21:55.443 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:55 smithi149 podman[58931]: 2024-04-22 23:21:55.203601866 +0000 UTC m=+0.757129752 container remove b221d725433a39f2cfdc3be4d9cb0740ed41b2dafa7c1c0fdc7b6aebf2a19cc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0) 2024-04-22T23:21:55.443 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:55 smithi149 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-22T23:21:55.443 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:55 smithi149 systemd[1]: Stopped Ceph osd.6 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:21:55.443 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:55 smithi149 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.6.service: Consumed 4min 59.809s CPU time. 2024-04-22T23:21:55.443 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:55 smithi149 systemd[1]: Starting Ceph osd.6 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:21:55.700 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:55 smithi149 podman[59030]: 2024-04-22 23:21:55.577330387 +0000 UTC m=+0.019502931 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:21:56.125 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:55 smithi149 podman[59030]: 2024-04-22 23:21:55.871658096 +0000 UTC m=+0.313830601 container create 4a528d779ea993c6a352e6a930bbee487b8452f7d8c544a9ec4908085d64ec43 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-22T23:21:56.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 270680 9.82 MB/sec execute 526 sec latency 59.204 ms 2024-04-22T23:21:56.404 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:56 smithi069 ceph-mon[63133]: Health check failed: Degraded data redundancy: 50/1608 objects degraded (3.109%), 6 pgs degraded (PG_DEGRADED) 2024-04-22T23:21:56.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:56 smithi149 ceph-mon[49797]: Health check failed: Degraded data redundancy: 50/1608 objects degraded (3.109%), 6 pgs degraded (PG_DEGRADED) 2024-04-22T23:21:56.451 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 podman[59030]: 2024-04-22 23:21:56.125003519 +0000 UTC m=+0.567176033 container init 4a528d779ea993c6a352e6a930bbee487b8452f7d8c544a9ec4908085d64ec43 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0) 2024-04-22T23:21:56.451 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 podman[59030]: 2024-04-22 23:21:56.128673303 +0000 UTC m=+0.570845814 container start 4a528d779ea993c6a352e6a930bbee487b8452f7d8c544a9ec4908085d64ec43 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-22T23:21:56.451 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 podman[59030]: 2024-04-22 23:21:56.250241368 +0000 UTC m=+0.692413874 container attach 4a528d779ea993c6a352e6a930bbee487b8452f7d8c544a9ec4908085d64ec43 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True) 2024-04-22T23:21:56.951 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate[59045]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:21:56.951 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate[59045]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:21:56.951 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 bash[59030]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:21:56.951 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 bash[59030]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:21:56.951 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 bash[59030]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T23:21:56.951 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 bash[59030]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T23:21:56.951 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 bash[59030]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T23:21:56.951 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 bash[59030]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T23:21:56.951 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 bash[59030]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-22T23:21:56.951 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 bash[59030]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T23:21:56.951 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 bash[59030]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-22T23:21:56.951 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate[59045]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T23:21:56.952 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate[59045]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T23:21:56.952 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate[59045]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T23:21:56.952 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate[59045]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T23:21:56.952 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate[59045]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-22T23:21:56.952 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate[59045]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T23:21:56.952 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate[59045]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-22T23:21:56.952 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:56 smithi149 podman[59030]: 2024-04-22 23:21:56.715299587 +0000 UTC m=+1.157472097 container died 4a528d779ea993c6a352e6a930bbee487b8452f7d8c544a9ec4908085d64ec43 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416) 2024-04-22T23:21:57.165 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:57 smithi069 ceph-mon[63133]: pgmap v256: 97 pgs: 2 active+undersized, 6 active+undersized+degraded, 4 peering, 6 stale+active+clean, 79 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.2 MiB/s wr, 284 op/s; 50/1608 objects degraded (3.109%) 2024-04-22T23:21:57.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 270976 9.81 MB/sec execute 527 sec latency 37.286 ms 2024-04-22T23:21:57.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:57 smithi149 ceph-mon[49797]: pgmap v256: 97 pgs: 2 active+undersized, 6 active+undersized+degraded, 4 peering, 6 stale+active+clean, 79 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.2 MiB/s wr, 284 op/s; 50/1608 objects degraded (3.109%) 2024-04-22T23:21:57.606 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:57 smithi149 podman[59177]: 2024-04-22 23:21:57.338214314 +0000 UTC m=+0.616928190 container remove 4a528d779ea993c6a352e6a930bbee487b8452f7d8c544a9ec4908085d64ec43 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416) 2024-04-22T23:21:57.606 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:57 smithi149 podman[59239]: 2024-04-22 23:21:57.469135834 +0000 UTC m=+0.022128520 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:21:57.867 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:57 smithi149 podman[59239]: 2024-04-22 23:21:57.605921605 +0000 UTC m=+0.158914320 container create b821ba1ee8121c30dd793cdbeb66e72eddce4f842042449e848ee40022ae059f (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-22T23:21:57.868 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:57 smithi149 podman[59239]: 2024-04-22 23:21:57.867178134 +0000 UTC m=+0.420170806 container init b821ba1ee8121c30dd793cdbeb66e72eddce4f842042449e848ee40022ae059f (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS) 2024-04-22T23:21:58.200 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:57 smithi149 podman[59239]: 2024-04-22 23:21:57.870423345 +0000 UTC m=+0.423416013 container start b821ba1ee8121c30dd793cdbeb66e72eddce4f842042449e848ee40022ae059f (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240416, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-22T23:21:58.200 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:57 smithi149 ceph-osd[59257]: -- 172.21.15.149:0/1809833750 <== mon.1 v2:172.21.15.149:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5579e5e7b680 con 0x5579e5e45000 2024-04-22T23:21:58.200 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:57 smithi149 bash[59239]: b821ba1ee8121c30dd793cdbeb66e72eddce4f842042449e848ee40022ae059f 2024-04-22T23:21:58.201 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:57 smithi149 systemd[1]: Started Ceph osd.6 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:21:58.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 271239 9.81 MB/sec execute 528 sec latency 49.934 ms 2024-04-22T23:21:58.700 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:21:58 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:21:58.416+0000 7f80a25d27c0 -1 Falling back to public interface 2024-04-22T23:21:59.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:59 smithi149 ceph-mon[49797]: pgmap v257: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 5.3 MiB/s wr, 506 op/s; 193/1662 objects degraded (11.613%) 2024-04-22T23:21:59.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:59 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:59.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:59 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:59.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:59 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:21:59.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:21:59 smithi149 ceph-mon[49797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T23:21:59.214 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:59 smithi069 ceph-mon[63133]: pgmap v257: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 5.3 MiB/s wr, 506 op/s; 193/1662 objects degraded (11.613%) 2024-04-22T23:21:59.214 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:59 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:59.214 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:59 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:21:59.215 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:59 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:21:59.215 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:21:59 smithi069 ceph-mon[63133]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T23:21:59.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 271639 9.82 MB/sec execute 529 sec latency 40.483 ms 2024-04-22T23:22:00.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 272028 9.81 MB/sec execute 530 sec latency 52.286 ms 2024-04-22T23:22:01.291 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:01 smithi069 ceph-mon[63133]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.8 MiB/s wr, 433 op/s; 193/1662 objects degraded (11.613%) 2024-04-22T23:22:01.292 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:01.292 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:22:01.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 272384 9.81 MB/sec execute 531 sec latency 41.473 ms 2024-04-22T23:22:01.321 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:01 smithi149 ceph-mon[49797]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.8 MiB/s wr, 433 op/s; 193/1662 objects degraded (11.613%) 2024-04-22T23:22:01.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:01.322 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:22:02.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 272715 9.81 MB/sec execute 532 sec latency 49.381 ms 2024-04-22T23:22:02.310 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:02 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 199/1656 objects degraded (12.017%), 25 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:02.352 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:02 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 199/1656 objects degraded (12.017%), 25 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:03.089 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:03 smithi069 ceph-mon[63133]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.3 MiB/s wr, 463 op/s; 199/1656 objects degraded (12.017%) 2024-04-22T23:22:03.089 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:03 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:03.089 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:03 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:03.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 273093 9.81 MB/sec execute 533 sec latency 41.049 ms 2024-04-22T23:22:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:03 smithi149 ceph-mon[49797]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.3 MiB/s wr, 463 op/s; 199/1656 objects degraded (12.017%) 2024-04-22T23:22:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:03 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:03 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:04.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 273472 9.82 MB/sec execute 534 sec latency 42.523 ms 2024-04-22T23:22:05.016 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:04 smithi149 ceph-mon[49797]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.2 MiB/s wr, 400 op/s; 193/1641 objects degraded (11.761%) 2024-04-22T23:22:05.016 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:04 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:05.016 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:04 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:05.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:04 smithi069 ceph-mon[63133]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.2 MiB/s wr, 400 op/s; 193/1641 objects degraded (11.761%) 2024-04-22T23:22:05.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:04 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:05.132 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:04 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:05.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 273871 9.82 MB/sec execute 535 sec latency 45.775 ms 2024-04-22T23:22:05.905 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:22:05 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:22:05.652+0000 7f80a25d27c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-22T23:22:06.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 274279 9.82 MB/sec execute 536 sec latency 40.297 ms 2024-04-22T23:22:07.160 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:07 smithi069 ceph-mon[63133]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 479 op/s; 197/1662 objects degraded (11.853%) 2024-04-22T23:22:07.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 274650 9.82 MB/sec execute 537 sec latency 48.893 ms 2024-04-22T23:22:07.338 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:07 smithi149 ceph-mon[49797]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 479 op/s; 197/1662 objects degraded (11.853%) 2024-04-22T23:22:07.700 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:22:07 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:22:07.336+0000 7f80a25d27c0 -1 osd.6 139 log_to_monitors true 2024-04-22T23:22:08.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:08 smithi149 ceph-mon[49797]: from='osd.6 [v2:172.21.15.149:6816/1119653428,v1:172.21.15.149:6817/1119653428]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T23:22:08.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:08 smithi149 ceph-mon[49797]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T23:22:08.202 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:08 smithi069 ceph-mon[63133]: from='osd.6 [v2:172.21.15.149:6816/1119653428,v1:172.21.15.149:6817/1119653428]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T23:22:08.202 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:08 smithi069 ceph-mon[63133]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T23:22:08.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 274934 9.81 MB/sec execute 538 sec latency 111.091 ms 2024-04-22T23:22:08.950 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:22:08 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:22:08.518+0000 7f808fa22700 -1 osd.6 139 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:22:09.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.8 MiB/s wr, 575 op/s; 202/1686 objects degraded (11.981%) 2024-04-22T23:22:09.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 202/1686 objects degraded (11.981%), 25 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:09.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T23:22:09.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='osd.6 [v2:172.21.15.149:6816/1119653428,v1:172.21.15.149:6817/1119653428]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:22:09.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: osdmap e142: 8 total, 7 up, 8 in 2024-04-22T23:22:09.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:22:09.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:09.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:09.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:22:09.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:22:09.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:09.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:09.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='osd.6 ' entity='osd.6' 2024-04-22T23:22:09.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:22:09.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:09.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:09.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:09.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:09 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T23:22:09.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 275281 9.81 MB/sec execute 539 sec latency 118.559 ms 2024-04-22T23:22:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.8 MiB/s wr, 575 op/s; 202/1686 objects degraded (11.981%) 2024-04-22T23:22:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 202/1686 objects degraded (11.981%), 25 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T23:22:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='osd.6 [v2:172.21.15.149:6816/1119653428,v1:172.21.15.149:6817/1119653428]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:22:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: osdmap e142: 8 total, 7 up, 8 in 2024-04-22T23:22:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:22:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:22:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:22:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:09.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='osd.6 ' entity='osd.6' 2024-04-22T23:22:09.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:22:09.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:09.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:09.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:09.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:09 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T23:22:10.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 275384 9.80 MB/sec execute 540 sec latency 774.891 ms 2024-04-22T23:22:10.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:10 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T23:22:10.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:10 smithi069 ceph-mon[63133]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-22T23:22:10.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:10 smithi069 ceph-mon[63133]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:22:10.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:10 smithi069 ceph-mon[63133]: osd.6 [v2:172.21.15.149:6816/1119653428,v1:172.21.15.149:6817/1119653428] boot 2024-04-22T23:22:10.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:10 smithi069 ceph-mon[63133]: osdmap e143: 8 total, 8 up, 8 in 2024-04-22T23:22:10.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:10 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:22:10.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:10 smithi069 ceph-mon[63133]: osdmap e144: 8 total, 8 up, 8 in 2024-04-22T23:22:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:10 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T23:22:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:10 smithi149 ceph-mon[49797]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-22T23:22:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:10 smithi149 ceph-mon[49797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:22:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:10 smithi149 ceph-mon[49797]: osd.6 [v2:172.21.15.149:6816/1119653428,v1:172.21.15.149:6817/1119653428] boot 2024-04-22T23:22:10.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:10 smithi149 ceph-mon[49797]: osdmap e143: 8 total, 8 up, 8 in 2024-04-22T23:22:10.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:10 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T23:22:10.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:10 smithi149 ceph-mon[49797]: osdmap e144: 8 total, 8 up, 8 in 2024-04-22T23:22:11.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 275394 9.78 MB/sec execute 541 sec latency 1730.112 ms 2024-04-22T23:22:11.318 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:11 smithi069 ceph-mon[63133]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.6 MiB/s wr, 439 op/s; 202/1686 objects degraded (11.981%) 2024-04-22T23:22:11.318 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:11 smithi069 ceph-mon[63133]: osdmap e145: 8 total, 8 up, 8 in 2024-04-22T23:22:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:11 smithi149 ceph-mon[49797]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.6 MiB/s wr, 439 op/s; 202/1686 objects degraded (11.981%) 2024-04-22T23:22:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:11 smithi149 ceph-mon[49797]: osdmap e145: 8 total, 8 up, 8 in 2024-04-22T23:22:11.725 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:22:12.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 275585 9.77 MB/sec execute 542 sec latency 244.226 ms 2024-04-22T23:22:12.519 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (8m) 2m ago 17m 23.2M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (5m) 2m ago 17m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (5m) 9s ago 16m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (8m) 2m ago 17m 57.1M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (11m) 2m ago 11m 38.4M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (11m) 9s ago 11m 15.5M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (7m) 2m ago 18m 509M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (6m) 9s ago 16m 452M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (6m) 2m ago 18m 51.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (6m) 9s ago 16m 42.5M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (11m) 2m ago 11m 264M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (11m) 9s ago 11m 86.9M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:22:12.520 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (8m) 2m ago 17m 17.0M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:22:12.521 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (8m) 9s ago 16m 19.4M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:22:12.521 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (4m) 2m ago 16m 345M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:22:12.521 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (3m) 2m ago 15m 283M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:22:12.521 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (2m) 2m ago 15m 210M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:22:12.521 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (2m) 2m ago 14m 12.7M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 e7772e3632e1 2024-04-22T23:22:12.521 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (91s) 9s ago 14m 298M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 036a1a8ba3c9 2024-04-22T23:22:12.521 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (53s) 9s ago 13m 269M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 539c90bd7194 2024-04-22T23:22:12.521 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (15s) 9s ago 13m 12.3M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 b821ba1ee812 2024-04-22T23:22:12.521 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (13m) 9s ago 13m 604M 4096M 17.2.0 e1d6a67b021e 49c48e51295f 2024-04-22T23:22:12.521 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (7m) 2m ago 16m 73.9M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:22:12.930 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:22:12.930 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:22:12.930 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:22:12.930 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:22:12.930 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:22:12.930 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:22:12.930 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:22:12.930 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:22:12.930 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T23:22:12.931 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 7 2024-04-22T23:22:12.931 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:22:12.931 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:22:12.931 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:22:12.931 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:22:12.931 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:22:12.931 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-22T23:22:12.931 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 11 2024-04-22T23:22:12.931 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:22:12.931 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:22:13.180 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:13 smithi069 ceph-mon[63133]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:13.181 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:13 smithi069 ceph-mon[63133]: pgmap v268: 97 pgs: 1 activating+undersized+degraded+remapped, 5 peering, 2 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+undersized+degraded, 73 active+clean; 349 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s wr, 369 op/s; 131/1692 objects degraded (7.742%); 122 KiB/s, 12 objects/s recovering 2024-04-22T23:22:13.181 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:13 smithi069 ceph-mon[63133]: osdmap e146: 8 total, 8 up, 8 in 2024-04-22T23:22:13.181 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:13 smithi069 ceph-mon[63133]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:13.181 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:13 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/1834113279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:13.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 275691 9.76 MB/sec execute 543 sec latency 1030.755 ms 2024-04-22T23:22:13.396 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:22:13.396 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:22:13.396 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:22:13.396 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:22:13.396 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:22:13.396 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:22:13.396 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:22:13.396 INFO:teuthology.orchestra.run.smithi069.stdout: "mon" 2024-04-22T23:22:13.397 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:22:13.397 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "13/23 daemons upgraded", 2024-04-22T23:22:13.397 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T23:22:13.397 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:22:13.397 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:22:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:13 smithi149 ceph-mon[49797]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:13 smithi149 ceph-mon[49797]: pgmap v268: 97 pgs: 1 activating+undersized+degraded+remapped, 5 peering, 2 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+undersized+degraded, 73 active+clean; 349 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s wr, 369 op/s; 131/1692 objects degraded (7.742%); 122 KiB/s, 12 objects/s recovering 2024-04-22T23:22:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:13 smithi149 ceph-mon[49797]: osdmap e146: 8 total, 8 up, 8 in 2024-04-22T23:22:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:13 smithi149 ceph-mon[49797]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:13 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/1834113279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:14.037 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN Degraded data redundancy: 131/1692 objects degraded (7.742%), 17 pgs degraded 2024-04-22T23:22:14.037 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 131/1692 objects degraded (7.742%), 17 pgs degraded 2024-04-22T23:22:14.037 INFO:teuthology.orchestra.run.smithi069.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-04-22T23:22:14.037 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-04-22T23:22:14.037 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-04-22T23:22:14.037 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-04-22T23:22:14.037 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-04-22T23:22:14.037 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-04-22T23:22:14.037 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-04-22T23:22:14.037 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1b is activating+undersized+degraded+remapped, acting [3,2] 2024-04-22T23:22:14.037 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-04-22T23:22:14.037 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-04-22T23:22:14.038 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2024-04-22T23:22:14.038 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-04-22T23:22:14.038 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-04-22T23:22:14.038 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-04-22T23:22:14.038 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-04-22T23:22:14.038 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-04-22T23:22:14.038 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-04-22T23:22:14.210 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:14 smithi069 ceph-mon[63133]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:14.211 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:14 smithi069 ceph-mon[63133]: osdmap e147: 8 total, 8 up, 8 in 2024-04-22T23:22:14.211 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:14 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/1540304879' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:22:14.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 275987 9.76 MB/sec execute 544 sec latency 36.510 ms 2024-04-22T23:22:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:14 smithi149 ceph-mon[49797]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:14 smithi149 ceph-mon[49797]: osdmap e147: 8 total, 8 up, 8 in 2024-04-22T23:22:14.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:14 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/1540304879' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:22:15.224 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:15 smithi069 ceph-mon[63133]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:15.224 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:15 smithi069 ceph-mon[63133]: pgmap v271: 97 pgs: 1 activating+undersized+degraded+remapped, 6 peering, 1 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.2 MiB/s wr, 426 op/s; 28/1602 objects degraded (1.748%); 1.3 MiB/s, 70 keys/s, 40 objects/s recovering 2024-04-22T23:22:15.224 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:15 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 28/1602 objects degraded (1.748%), 2 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:15.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 276368 9.75 MB/sec execute 545 sec latency 36.089 ms 2024-04-22T23:22:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:15 smithi149 ceph-mon[49797]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:15 smithi149 ceph-mon[49797]: pgmap v271: 97 pgs: 1 activating+undersized+degraded+remapped, 6 peering, 1 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.2 MiB/s wr, 426 op/s; 28/1602 objects degraded (1.748%); 1.3 MiB/s, 70 keys/s, 40 objects/s recovering 2024-04-22T23:22:15.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:15 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 28/1602 objects degraded (1.748%), 2 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:16.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 276710 9.75 MB/sec execute 546 sec latency 45.026 ms 2024-04-22T23:22:17.023 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:16 smithi069 ceph-mon[63133]: pgmap v272: 97 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 2.3 MiB/s wr, 269 op/s; 28/1608 objects degraded (1.741%); 1.1 MiB/s, 49 keys/s, 37 objects/s recovering 2024-04-22T23:22:17.024 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:17.024 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:22:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:16 smithi149 ceph-mon[49797]: pgmap v272: 97 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 2.3 MiB/s wr, 269 op/s; 28/1608 objects degraded (1.741%); 1.1 MiB/s, 49 keys/s, 37 objects/s recovering 2024-04-22T23:22:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:17.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:22:17.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 277061 9.76 MB/sec execute 547 sec latency 46.702 ms 2024-04-22T23:22:18.033 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:17 smithi069 ceph-mon[63133]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/1608 objects degraded (1.741%), 2 pgs degraded) 2024-04-22T23:22:18.033 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:17 smithi069 ceph-mon[63133]: Cluster is now healthy 2024-04-22T23:22:18.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:17 smithi149 ceph-mon[49797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/1608 objects degraded (1.741%), 2 pgs degraded) 2024-04-22T23:22:18.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:17 smithi149 ceph-mon[49797]: Cluster is now healthy 2024-04-22T23:22:18.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 277384 9.76 MB/sec execute 548 sec latency 56.259 ms 2024-04-22T23:22:19.055 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:18 smithi069 ceph-mon[63133]: pgmap v273: 97 pgs: 97 active+clean; 350 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 480 op/s; 2.1 MiB/s, 42 keys/s, 35 objects/s recovering 2024-04-22T23:22:19.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:18 smithi149 ceph-mon[49797]: pgmap v273: 97 pgs: 97 active+clean; 350 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 480 op/s; 2.1 MiB/s, 42 keys/s, 35 objects/s recovering 2024-04-22T23:22:19.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 277753 9.76 MB/sec execute 549 sec latency 47.862 ms 2024-04-22T23:22:20.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 278045 9.75 MB/sec execute 550 sec latency 43.687 ms 2024-04-22T23:22:21.098 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:20 smithi069 ceph-mon[63133]: pgmap v274: 97 pgs: 97 active+clean; 350 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 2.8 MiB/s wr, 267 op/s; 1.7 MiB/s, 35 keys/s, 23 objects/s recovering 2024-04-22T23:22:21.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:20 smithi149 ceph-mon[49797]: pgmap v274: 97 pgs: 97 active+clean; 350 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 2.8 MiB/s wr, 267 op/s; 1.7 MiB/s, 35 keys/s, 23 objects/s recovering 2024-04-22T23:22:21.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 278312 9.74 MB/sec execute 551 sec latency 53.638 ms 2024-04-22T23:22:22.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 278669 9.75 MB/sec execute 552 sec latency 45.653 ms 2024-04-22T23:22:22.883 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:22 smithi069 ceph-mon[63133]: pgmap v275: 97 pgs: 97 active+clean; 348 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.8 MiB/s wr, 425 op/s; 1004 KiB/s, 29 keys/s, 15 objects/s recovering 2024-04-22T23:22:23.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:22 smithi149 ceph-mon[49797]: pgmap v275: 97 pgs: 97 active+clean; 348 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.8 MiB/s wr, 425 op/s; 1004 KiB/s, 29 keys/s, 15 objects/s recovering 2024-04-22T23:22:23.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 279040 9.75 MB/sec execute 553 sec latency 50.163 ms 2024-04-22T23:22:23.901 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:23 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T23:22:24.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:23 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T23:22:24.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 279354 9.74 MB/sec execute 554 sec latency 53.288 ms 2024-04-22T23:22:24.925 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:24 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T23:22:24.925 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:24 smithi069 ceph-mon[63133]: Upgrade: osd.7 is safe to restart 2024-04-22T23:22:24.925 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:24 smithi069 ceph-mon[63133]: pgmap v276: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 410 op/s; 875 KiB/s, 11 objects/s recovering 2024-04-22T23:22:25.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:24 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T23:22:25.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:24 smithi149 ceph-mon[49797]: Upgrade: osd.7 is safe to restart 2024-04-22T23:22:25.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:24 smithi149 ceph-mon[49797]: pgmap v276: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 410 op/s; 875 KiB/s, 11 objects/s recovering 2024-04-22T23:22:25.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 279656 9.74 MB/sec execute 555 sec latency 51.008 ms 2024-04-22T23:22:26.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 279934 9.73 MB/sec execute 556 sec latency 51.965 ms 2024-04-22T23:22:26.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:26 smithi069 ceph-mon[63133]: Upgrade: Updating osd.7 2024-04-22T23:22:26.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:26 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:26.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:26 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T23:22:26.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:26 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:22:26.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:26 smithi069 ceph-mon[63133]: Deploying daemon osd.7 on smithi149 2024-04-22T23:22:26.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:26 smithi069 ceph-mon[63133]: pgmap v277: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 389 op/s; 748 KiB/s, 6 objects/s recovering 2024-04-22T23:22:26.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:26 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:26.919 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:26 smithi149 ceph-mon[49797]: Upgrade: Updating osd.7 2024-04-22T23:22:26.920 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:26 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:26.920 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:26 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T23:22:26.920 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:26 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:22:26.920 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:26 smithi149 ceph-mon[49797]: Deploying daemon osd.7 on smithi149 2024-04-22T23:22:26.920 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:26 smithi149 ceph-mon[49797]: pgmap v277: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 389 op/s; 748 KiB/s, 6 objects/s recovering 2024-04-22T23:22:26.920 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:26 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:27.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 280217 9.73 MB/sec execute 557 sec latency 54.948 ms 2024-04-22T23:22:28.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 280598 9.74 MB/sec execute 558 sec latency 35.176 ms 2024-04-22T23:22:28.885 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:28 smithi069 ceph-mon[63133]: pgmap v278: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.3 MiB/s wr, 556 op/s; 659 KiB/s, 2 objects/s recovering 2024-04-22T23:22:29.019 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:28 smithi149 ceph-mon[49797]: pgmap v278: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.3 MiB/s wr, 556 op/s; 659 KiB/s, 2 objects/s recovering 2024-04-22T23:22:29.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 280932 9.74 MB/sec execute 559 sec latency 43.864 ms 2024-04-22T23:22:29.700 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:29 smithi149 systemd[1]: Stopping Ceph osd.7 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:22:29.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:29 smithi069 ceph-mon[63133]: osd.7 marked itself down and dead 2024-04-22T23:22:30.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:29 smithi149 ceph-mon[49797]: osd.7 marked itself down and dead 2024-04-22T23:22:30.200 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:29 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[37197]: 2024-04-22T23:22:29.715+0000 7f4a4fb5d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:22:30.201 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:29 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[37197]: 2024-04-22T23:22:29.715+0000 7f4a4fb5d700 -1 osd.7 147 *** Got signal Terminated *** 2024-04-22T23:22:30.201 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:29 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[37197]: 2024-04-22T23:22:29.715+0000 7f4a4fb5d700 -1 osd.7 147 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:22:30.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 281246 9.73 MB/sec execute 560 sec latency 408.127 ms 2024-04-22T23:22:30.803 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:30 smithi149 podman[61079]: 2024-04-22 23:22:30.540504946 +0000 UTC m=+0.962463299 container died 49c48e51295f9a04a4eee8ce5c1331f38eb07ec4c39deb05cc958ee7f65abc36 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, name=centos-stream, release=754, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True) 2024-04-22T23:22:31.066 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:31 smithi149 podman[61079]: 2024-04-22 23:22:31.04842738 +0000 UTC m=+1.470385739 container cleanup 49c48e51295f9a04a4eee8ce5c1331f38eb07ec4c39deb05cc958ee7f65abc36 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, release=754, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T23:22:31.066 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:31 smithi149 bash[61079]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7 2024-04-22T23:22:31.067 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:30 smithi149 ceph-mon[49797]: pgmap v279: 97 pgs: 97 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.1 MiB/s wr, 420 op/s 2024-04-22T23:22:31.067 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:30 smithi149 ceph-mon[49797]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:22:31.067 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:30 smithi149 ceph-mon[49797]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-22T23:22:31.067 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:30 smithi149 ceph-mon[49797]: osdmap e148: 8 total, 7 up, 8 in 2024-04-22T23:22:31.067 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:31.067 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:22:31.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:30 smithi069 ceph-mon[63133]: pgmap v279: 97 pgs: 97 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.1 MiB/s wr, 420 op/s 2024-04-22T23:22:31.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:30 smithi069 ceph-mon[63133]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T23:22:31.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:30 smithi069 ceph-mon[63133]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-22T23:22:31.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:30 smithi069 ceph-mon[63133]: osdmap e148: 8 total, 7 up, 8 in 2024-04-22T23:22:31.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:31.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:22:31.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 281412 9.72 MB/sec execute 561 sec latency 1061.606 ms 2024-04-22T23:22:31.877 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:31 smithi149 podman[61096]: 2024-04-22 23:22:31.60020497 +0000 UTC m=+1.053392879 container remove 49c48e51295f9a04a4eee8ce5c1331f38eb07ec4c39deb05cc958ee7f65abc36 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, ceph=True, maintainer=Guillaume Abrioux , version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream) 2024-04-22T23:22:32.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:31 smithi069 ceph-mon[63133]: osdmap e149: 8 total, 7 up, 8 in 2024-04-22T23:22:32.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:31 smithi149 ceph-mon[49797]: osdmap e149: 8 total, 7 up, 8 in 2024-04-22T23:22:32.200 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:32 smithi149 podman[61184]: 2024-04-22 23:22:31.909973614 +0000 UTC m=+0.020346602 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:22:32.201 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:32 smithi149 podman[61184]: 2024-04-22 23:22:32.084703582 +0000 UTC m=+0.195076571 container create 6c8e8f6232427aef8ed06d5a6a66797ad6fff6a2a34ab4597e45fe8abaab7c28 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-22T23:22:32.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 281814 9.72 MB/sec execute 562 sec latency 31.687 ms 2024-04-22T23:22:32.465 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:32 smithi149 podman[61184]: 2024-04-22 23:22:32.296058698 +0000 UTC m=+0.406431681 container init 6c8e8f6232427aef8ed06d5a6a66797ad6fff6a2a34ab4597e45fe8abaab7c28 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-22T23:22:32.466 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:32 smithi149 podman[61184]: 2024-04-22 23:22:32.300318691 +0000 UTC m=+0.410691678 container start 6c8e8f6232427aef8ed06d5a6a66797ad6fff6a2a34ab4597e45fe8abaab7c28 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T23:22:32.466 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:32 smithi149 podman[61184]: 2024-04-22 23:22:32.465177126 +0000 UTC m=+0.575550123 container attach 6c8e8f6232427aef8ed06d5a6a66797ad6fff6a2a34ab4597e45fe8abaab7c28 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=) 2024-04-22T23:22:32.878 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:32 smithi149 podman[61184]: 2024-04-22 23:22:32.648317884 +0000 UTC m=+0.758690867 container died 6c8e8f6232427aef8ed06d5a6a66797ad6fff6a2a34ab4597e45fe8abaab7c28 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416) 2024-04-22T23:22:32.892 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:32 smithi069 ceph-mon[63133]: pgmap v282: 97 pgs: 2 active+undersized, 13 active+undersized+degraded, 17 peering, 2 stale+active+clean, 63 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 514 op/s; 133/1638 objects degraded (8.120%) 2024-04-22T23:22:32.892 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:32 smithi069 ceph-mon[63133]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-22T23:22:32.893 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:32 smithi069 ceph-mon[63133]: Health check failed: Degraded data redundancy: 133/1638 objects degraded (8.120%), 13 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:32 smithi149 ceph-mon[49797]: pgmap v282: 97 pgs: 2 active+undersized, 13 active+undersized+degraded, 17 peering, 2 stale+active+clean, 63 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 514 op/s; 133/1638 objects degraded (8.120%) 2024-04-22T23:22:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:32 smithi149 ceph-mon[49797]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-22T23:22:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:32 smithi149 ceph-mon[49797]: Health check failed: Degraded data redundancy: 133/1638 objects degraded (8.120%), 13 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:33.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 282235 9.73 MB/sec execute 563 sec latency 38.145 ms 2024-04-22T23:22:33.649 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:33 smithi149 podman[61209]: 2024-04-22 23:22:33.317771609 +0000 UTC m=+0.663309069 container remove 6c8e8f6232427aef8ed06d5a6a66797ad6fff6a2a34ab4597e45fe8abaab7c28 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-22T23:22:33.650 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:33 smithi149 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.7.service: Deactivated successfully. 2024-04-22T23:22:33.650 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:33 smithi149 systemd[1]: Stopped Ceph osd.7 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:22:33.650 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:33 smithi149 systemd[1]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.7.service: Consumed 7min 11.905s CPU time. 2024-04-22T23:22:33.928 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:33 smithi149 systemd[1]: Starting Ceph osd.7 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:22:33.928 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:33 smithi149 podman[61307]: 2024-04-22 23:22:33.795668347 +0000 UTC m=+0.031254272 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:22:34.200 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:33 smithi149 podman[61307]: 2024-04-22 23:22:33.927578183 +0000 UTC m=+0.163164125 container create e76fce3c38128842b899caa3d2e7b5a51781fd985ee226eafe6ee2da25c416a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T23:22:34.200 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 podman[61307]: 2024-04-22 23:22:34.17237828 +0000 UTC m=+0.407964181 container init e76fce3c38128842b899caa3d2e7b5a51781fd985ee226eafe6ee2da25c416a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T23:22:34.200 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 podman[61307]: 2024-04-22 23:22:34.17673132 +0000 UTC m=+0.412317223 container start e76fce3c38128842b899caa3d2e7b5a51781fd985ee226eafe6ee2da25c416a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-22T23:22:34.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 282671 9.73 MB/sec execute 564 sec latency 37.428 ms 2024-04-22T23:22:34.691 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 podman[61307]: 2024-04-22 23:22:34.281097744 +0000 UTC m=+0.516683668 container attach e76fce3c38128842b899caa3d2e7b5a51781fd985ee226eafe6ee2da25c416a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, ceph=True, GIT_CLEAN=True) 2024-04-22T23:22:34.928 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:34 smithi069 ceph-mon[63133]: pgmap v283: 97 pgs: 2 active+undersized, 23 active+undersized+degraded, 17 peering, 55 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.8 MiB/s wr, 520 op/s; 170/1647 objects degraded (10.322%) 2024-04-22T23:22:34.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:34 smithi149 ceph-mon[49797]: pgmap v283: 97 pgs: 2 active+undersized, 23 active+undersized+degraded, 17 peering, 55 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.8 MiB/s wr, 520 op/s; 170/1647 objects degraded (10.322%) 2024-04-22T23:22:34.950 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate[61322]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:22:34.950 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate[61322]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:22:34.950 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 bash[61307]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 bash[61307]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 bash[61307]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 bash[61307]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 bash[61307]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 bash[61307]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 bash[61307]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 bash[61307]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 bash[61307]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate[61322]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate[61322]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate[61322]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate[61322]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate[61322]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate[61322]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T23:22:34.951 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate[61322]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-22T23:22:34.952 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:34 smithi149 podman[61307]: 2024-04-22 23:22:34.717853873 +0000 UTC m=+0.953439765 container died e76fce3c38128842b899caa3d2e7b5a51781fd985ee226eafe6ee2da25c416a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-22T23:22:35.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 282973 9.72 MB/sec execute 565 sec latency 52.896 ms 2024-04-22T23:22:35.700 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:35 smithi149 podman[61445]: 2024-04-22 23:22:35.368709853 +0000 UTC m=+0.640525259 container remove e76fce3c38128842b899caa3d2e7b5a51781fd985ee226eafe6ee2da25c416a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-22T23:22:35.700 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:35 smithi149 podman[61506]: 2024-04-22 23:22:35.49191604 +0000 UTC m=+0.019078210 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:22:35.700 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:35 smithi149 podman[61506]: 2024-04-22 23:22:35.611335199 +0000 UTC m=+0.138497344 container create fec23dae1ff2a68ed24b37526311cd0d590fa92ce6f94bf531800b2507caec51 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-22T23:22:35.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:35 smithi069 ceph-mon[63133]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-04-22T23:22:36.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:35 smithi149 ceph-mon[49797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-04-22T23:22:36.200 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:35 smithi149 podman[61506]: 2024-04-22 23:22:35.839375557 +0000 UTC m=+0.366537699 container init fec23dae1ff2a68ed24b37526311cd0d590fa92ce6f94bf531800b2507caec51 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T23:22:36.201 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:35 smithi149 podman[61506]: 2024-04-22 23:22:35.842586971 +0000 UTC m=+0.369749122 container start fec23dae1ff2a68ed24b37526311cd0d590fa92ce6f94bf531800b2507caec51 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-22T23:22:36.201 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:35 smithi149 ceph-osd[61525]: -- 172.21.15.149:0/2153259268 <== mon.1 v2:172.21.15.149:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fce622d680 con 0x55fce61f6c00 2024-04-22T23:22:36.201 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:35 smithi149 bash[61506]: fec23dae1ff2a68ed24b37526311cd0d590fa92ce6f94bf531800b2507caec51 2024-04-22T23:22:36.201 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:35 smithi149 systemd[1]: Started Ceph osd.7 for 6928ed7e-00fc-11ef-bc93-c7b262605968. 2024-04-22T23:22:36.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 283281 9.73 MB/sec execute 566 sec latency 81.887 ms 2024-04-22T23:22:36.700 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:36 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:22:36.387+0000 7effdd4d87c0 -1 Falling back to public interface 2024-04-22T23:22:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:36 smithi149 ceph-mon[49797]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.6 MiB/s wr, 312 op/s; 247/1638 objects degraded (15.079%) 2024-04-22T23:22:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:36 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:36 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:36 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:22:37.244 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:36 smithi069 ceph-mon[63133]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.6 MiB/s wr, 312 op/s; 247/1638 objects degraded (15.079%) 2024-04-22T23:22:37.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:36 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:37.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:36 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:37.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:36 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:22:37.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 283651 9.72 MB/sec execute 567 sec latency 52.675 ms 2024-04-22T23:22:38.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:38 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 245/1644 objects degraded (14.903%), 35 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:38.276 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:38 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 245/1644 objects degraded (14.903%), 35 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:38.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 283939 9.73 MB/sec execute 568 sec latency 68.301 ms 2024-04-22T23:22:39.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 284226 9.73 MB/sec execute 569 sec latency 44.603 ms 2024-04-22T23:22:39.303 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:39 smithi069 ceph-mon[63133]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 MiB/s wr, 622 op/s; 245/1644 objects degraded (14.903%) 2024-04-22T23:22:39.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:39 smithi149 ceph-mon[49797]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 MiB/s wr, 622 op/s; 245/1644 objects degraded (14.903%) 2024-04-22T23:22:40.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 284475 9.72 MB/sec execute 570 sec latency 85.265 ms 2024-04-22T23:22:41.082 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:41 smithi069 ceph-mon[63133]: pgmap v286: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.9 MiB/s wr, 508 op/s; 245/1638 objects degraded (14.957%) 2024-04-22T23:22:41.083 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:41 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:41.083 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:41 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:41.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 284824 9.71 MB/sec execute 571 sec latency 61.440 ms 2024-04-22T23:22:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:41 smithi149 ceph-mon[49797]: pgmap v286: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.9 MiB/s wr, 508 op/s; 245/1638 objects degraded (14.957%) 2024-04-22T23:22:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:41 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:41.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:41 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:42.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 285097 9.70 MB/sec execute 572 sec latency 60.933 ms 2024-04-22T23:22:43.173 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:43 smithi069 ceph-mon[63133]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.1 MiB/s wr, 426 op/s; 259/1662 objects degraded (15.584%) 2024-04-22T23:22:43.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:43.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:43.486 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:43 smithi149 ceph-mon[49797]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.1 MiB/s wr, 426 op/s; 259/1662 objects degraded (15.584%) 2024-04-22T23:22:43.486 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:43.486 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:43.486 INFO:teuthology.orchestra.run.smithi069.stdout: 5 285377 9.70 MB/sec execute 573 sec latency 39.934 ms 2024-04-22T23:22:44.186 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:44 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 260/1647 objects degraded (15.786%), 35 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:44.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:44 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 260/1647 objects degraded (15.786%), 35 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:44.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 285739 9.71 MB/sec execute 574 sec latency 44.250 ms 2024-04-22T23:22:44.487 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:22:45.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:45 smithi069 ceph-mon[63133]: pgmap v288: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 4.8 MiB/s wr, 499 op/s; 260/1647 objects degraded (15.786%) 2024-04-22T23:22:45.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 286105 9.70 MB/sec execute 575 sec latency 73.928 ms 2024-04-22T23:22:45.348 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:22:45.348 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (8m) 2m ago 18m 23.2M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:22:45.348 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (5m) 2m ago 18m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:22:45.348 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (5m) 4s ago 17m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:22:45.348 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (8m) 2m ago 17m 57.1M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:22:45.348 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (12m) 2m ago 12m 38.4M - 17.2.0 e1d6a67b021e 8038fd37be07 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (12m) 4s ago 12m 15.6M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (8m) 2m ago 19m 509M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (7m) 4s ago 17m 453M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (7m) 2m ago 19m 51.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (6m) 4s ago 17m 42.6M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (11m) 2m ago 11m 264M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (11m) 4s ago 11m 86.9M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (8m) 2m ago 17m 17.0M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (8m) 4s ago 17m 19.2M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (5m) 2m ago 16m 345M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (4m) 2m ago 16m 283M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (3m) 2m ago 15m 210M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (2m) 2m ago 15m 12.7M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 e7772e3632e1 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (2m) 4s ago 14m 334M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 036a1a8ba3c9 2024-04-22T23:22:45.349 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (86s) 4s ago 14m 284M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 539c90bd7194 2024-04-22T23:22:45.350 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (47s) 4s ago 14m 278M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 b821ba1ee812 2024-04-22T23:22:45.350 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (9s) 4s ago 13m 15.1M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 fec23dae1ff2 2024-04-22T23:22:45.350 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (7m) 2m ago 16m 73.9M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:22:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:45 smithi149 ceph-mon[49797]: pgmap v288: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 4.8 MiB/s wr, 499 op/s; 260/1647 objects degraded (15.786%) 2024-04-22T23:22:45.919 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:22:45.919 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:22:45.919 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 7 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 11 2024-04-22T23:22:45.920 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:22:45.921 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:22:46.228 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:46.228 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:46.228 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/1751363499' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:46.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-22T23:22:46.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-22T23:22:46.231 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-22T23:22:46.231 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-22T23:22:46.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 286385 9.70 MB/sec execute 576 sec latency 75.339 ms 2024-04-22T23:22:46.347 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:22:46.347 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:22:46.347 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:22:46.347 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:22:46.347 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:22:46.347 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:22:46.347 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:22:46.348 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-04-22T23:22:46.348 INFO:teuthology.orchestra.run.smithi069.stdout: "osd" 2024-04-22T23:22:46.348 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:22:46.348 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "14/23 daemons upgraded", 2024-04-22T23:22:46.348 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T23:22:46.348 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:22:46.348 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:22:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:22:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/1751363499' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:22:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:22:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-22T23:22:46.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-22T23:22:46.453 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-22T23:22:46.453 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-22T23:22:46.453 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-22T23:22:46.453 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-22T23:22:46.453 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-22T23:22:46.453 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-22T23:22:46.835 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 258/1623 objects degraded (15.896%), 35 pgs degraded 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout: osd.7 (root=default,host=smithi149) is down 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 258/1623 objects degraded (15.896%), 35 pgs degraded 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-04-22T23:22:46.836 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.7 is active+undersized+degraded, acting [3,0] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.8 is active+undersized+degraded, acting [3,1] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-04-22T23:22:46.837 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-04-22T23:22:46.838 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.f is active+undersized+degraded, acting [4,0] 2024-04-22T23:22:46.838 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-04-22T23:22:46.838 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-04-22T23:22:46.838 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-04-22T23:22:46.838 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-04-22T23:22:46.838 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-04-22T23:22:46.838 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-04-22T23:22:46.838 INFO:teuthology.orchestra.run.smithi069.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-04-22T23:22:46.838 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-04-22T23:22:46.838 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-04-22T23:22:46.838 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-04-22T23:22:46.838 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.14 is active+undersized+degraded, acting [3,1] 2024-04-22T23:22:47.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:47 smithi069 ceph-mon[63133]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.9 MiB/s wr, 495 op/s; 258/1623 objects degraded (15.896%) 2024-04-22T23:22:47.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:47 smithi069 ceph-mon[63133]: Upgrade: Setting container_image for all osd 2024-04-22T23:22:47.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:47 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-22T23:22:47.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:47 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-22T23:22:47.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:47 smithi069 ceph-mon[63133]: Upgrade: Setting require_osd_release to 19 squid 2024-04-22T23:22:47.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:47 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-22T23:22:47.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:47 smithi069 ceph-mon[63133]: from='client.34454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:47.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:47 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/2798335828' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:22:47.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 286631 9.70 MB/sec execute 577 sec latency 45.788 ms 2024-04-22T23:22:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:47 smithi149 ceph-mon[49797]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.9 MiB/s wr, 495 op/s; 258/1623 objects degraded (15.896%) 2024-04-22T23:22:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:47 smithi149 ceph-mon[49797]: Upgrade: Setting container_image for all osd 2024-04-22T23:22:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:47 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-22T23:22:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:47 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-22T23:22:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:47 smithi149 ceph-mon[49797]: Upgrade: Setting require_osd_release to 19 squid 2024-04-22T23:22:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:47 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-22T23:22:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:47 smithi149 ceph-mon[49797]: from='client.34454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:22:47.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:47 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/2798335828' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:22:47.451 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:47 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:22:47.029+0000 7effdd4d87c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-22T23:22:48.300 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:48 smithi069 ceph-mon[63133]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-22T23:22:48.300 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:48 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-22T23:22:48.300 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:48 smithi069 ceph-mon[63133]: osdmap e150: 8 total, 7 up, 8 in 2024-04-22T23:22:48.300 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:48 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi069.jiyres"]}]: dispatch 2024-04-22T23:22:48.300 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:48 smithi069 ceph-mon[63133]: Upgrade: It appears safe to stop mds.foofs.smithi069.jiyres 2024-04-22T23:22:48.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 286970 9.70 MB/sec execute 578 sec latency 63.990 ms 2024-04-22T23:22:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:48 smithi149 ceph-mon[49797]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-22T23:22:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:48 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-22T23:22:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:48 smithi149 ceph-mon[49797]: osdmap e150: 8 total, 7 up, 8 in 2024-04-22T23:22:48.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:48 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi069.jiyres"]}]: dispatch 2024-04-22T23:22:48.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:48 smithi149 ceph-mon[49797]: Upgrade: It appears safe to stop mds.foofs.smithi069.jiyres 2024-04-22T23:22:49.195 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:48 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:22:48.930+0000 7effdd4d87c0 -1 osd.7 147 log_to_monitors true 2024-04-22T23:22:49.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 287283 9.69 MB/sec execute 579 sec latency 122.869 ms 2024-04-22T23:22:49.315 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:49 smithi069 ceph-mon[63133]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 340 op/s; 248/1608 objects degraded (15.423%) 2024-04-22T23:22:49.316 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:49 smithi069 ceph-mon[63133]: from='osd.7 [v2:172.21.15.149:6824/3602069127,v1:172.21.15.149:6825/3602069127]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T23:22:49.316 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:49 smithi069 ceph-mon[63133]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T23:22:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:49 smithi149 ceph-mon[49797]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 340 op/s; 248/1608 objects degraded (15.423%) 2024-04-22T23:22:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:49 smithi149 ceph-mon[49797]: from='osd.7 [v2:172.21.15.149:6824/3602069127,v1:172.21.15.149:6825/3602069127]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T23:22:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:49 smithi149 ceph-mon[49797]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T23:22:49.950 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:49 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:22:49.665+0000 7effca928700 -1 osd.7 147 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T23:22:50.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 287488 9.69 MB/sec execute 580 sec latency 158.799 ms 2024-04-22T23:22:50.360 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:50 smithi069 ceph-mon[63133]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T23:22:50.360 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:50 smithi069 ceph-mon[63133]: from='osd.7 [v2:172.21.15.149:6824/3602069127,v1:172.21.15.149:6825/3602069127]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:22:50.360 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:50 smithi069 ceph-mon[63133]: osdmap e151: 8 total, 7 up, 8 in 2024-04-22T23:22:50.360 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:50 smithi069 ceph-mon[63133]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:22:50.360 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:50 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:50.360 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:50 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.jiyres", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T23:22:50.361 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:50 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:22:50.361 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:50 smithi069 ceph-mon[63133]: from='osd.7 ' entity='osd.7' 2024-04-22T23:22:50.361 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:50 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 248/1608 objects degraded (15.423%), 35 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:50.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:50 smithi149 ceph-mon[49797]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T23:22:50.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:50 smithi149 ceph-mon[49797]: from='osd.7 [v2:172.21.15.149:6824/3602069127,v1:172.21.15.149:6825/3602069127]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:22:50.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:50 smithi149 ceph-mon[49797]: osdmap e151: 8 total, 7 up, 8 in 2024-04-22T23:22:50.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:50 smithi149 ceph-mon[49797]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2024-04-22T23:22:50.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:50 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:22:50.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:50 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.jiyres", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T23:22:50.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:50 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:22:50.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:50 smithi149 ceph-mon[49797]: from='osd.7 ' entity='osd.7' 2024-04-22T23:22:50.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:50 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 248/1608 objects degraded (15.423%), 35 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:51.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 287642 9.68 MB/sec execute 581 sec latency 641.714 ms 2024-04-22T23:22:51.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:51 smithi069 ceph-mon[63133]: Upgrade: Updating mds.foofs.smithi069.jiyres 2024-04-22T23:22:51.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:51 smithi069 ceph-mon[63133]: Deploying daemon mds.foofs.smithi069.jiyres on smithi069 2024-04-22T23:22:51.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:51 smithi069 ceph-mon[63133]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.0 MiB/s wr, 522 op/s; 249/1605 objects degraded (15.514%) 2024-04-22T23:22:51.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:51 smithi069 ceph-mon[63133]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:22:51.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:51 smithi069 ceph-mon[63133]: osd.7 [v2:172.21.15.149:6824/3602069127,v1:172.21.15.149:6825/3602069127] boot 2024-04-22T23:22:51.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:51 smithi069 ceph-mon[63133]: osdmap e152: 8 total, 8 up, 8 in 2024-04-22T23:22:51.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:22:51.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:51 smithi149 ceph-mon[49797]: Upgrade: Updating mds.foofs.smithi069.jiyres 2024-04-22T23:22:51.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:51 smithi149 ceph-mon[49797]: Deploying daemon mds.foofs.smithi069.jiyres on smithi069 2024-04-22T23:22:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:51 smithi149 ceph-mon[49797]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.0 MiB/s wr, 522 op/s; 249/1605 objects degraded (15.514%) 2024-04-22T23:22:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:51 smithi149 ceph-mon[49797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T23:22:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:51 smithi149 ceph-mon[49797]: osd.7 [v2:172.21.15.149:6824/3602069127,v1:172.21.15.149:6825/3602069127] boot 2024-04-22T23:22:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:51 smithi149 ceph-mon[49797]: osdmap e152: 8 total, 8 up, 8 in 2024-04-22T23:22:51.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T23:22:52.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 287669 9.66 MB/sec execute 582 sec latency 1642.107 ms 2024-04-22T23:22:52.908 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:52 smithi069 ceph-mon[63133]: osdmap e153: 8 total, 8 up, 8 in 2024-04-22T23:22:52.909 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:52 smithi069 ceph-mon[63133]: pgmap v296: 97 pgs: 9 peering, 5 active+undersized, 28 active+undersized+degraded, 55 active+clean; 351 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 594 op/s; 187/1614 objects degraded (11.586%) 2024-04-22T23:22:52.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:52 smithi149 ceph-mon[49797]: osdmap e153: 8 total, 8 up, 8 in 2024-04-22T23:22:52.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:52 smithi149 ceph-mon[49797]: pgmap v296: 97 pgs: 9 peering, 5 active+undersized, 28 active+undersized+degraded, 55 active+clean; 351 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 594 op/s; 187/1614 objects degraded (11.586%) 2024-04-22T23:22:53.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 287684 9.65 MB/sec execute 583 sec latency 2193.789 ms 2024-04-22T23:22:53.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:52 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069[63129]: 2024-04-22T23:22:52.968+0000 7f4e9d8c7700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T23:22:53.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:53 smithi149 ceph-mon[49797]: osdmap e154: 8 total, 8 up, 8 in 2024-04-22T23:22:53.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:53 smithi149 ceph-mon[49797]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-22T23:22:53.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:53 smithi149 ceph-mon[49797]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T23:22:53.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:53 smithi149 ceph-mon[49797]: osdmap e155: 8 total, 8 up, 8 in 2024-04-22T23:22:53.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:53 smithi149 ceph-mon[49797]: Standby daemon mds.foofs.smithi149.ickhha assigned to filesystem foofs as rank 0 2024-04-22T23:22:53.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:53 smithi149 ceph-mon[49797]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-22T23:22:53.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:53 smithi149 ceph-mon[49797]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-22T23:22:53.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:53 smithi149 ceph-mon[49797]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-22T23:22:53.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:53 smithi149 ceph-mon[49797]: fsmap foofs:1/1 {0=foofs.smithi149.ickhha=up:replay} 2024-04-22T23:22:54.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:53 smithi069 ceph-mon[63133]: osdmap e154: 8 total, 8 up, 8 in 2024-04-22T23:22:54.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:53 smithi069 ceph-mon[63133]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-22T23:22:54.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:53 smithi069 ceph-mon[63133]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T23:22:54.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:53 smithi069 ceph-mon[63133]: osdmap e155: 8 total, 8 up, 8 in 2024-04-22T23:22:54.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:53 smithi069 ceph-mon[63133]: Standby daemon mds.foofs.smithi149.ickhha assigned to filesystem foofs as rank 0 2024-04-22T23:22:54.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:53 smithi069 ceph-mon[63133]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-22T23:22:54.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:53 smithi069 ceph-mon[63133]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-22T23:22:54.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:53 smithi069 ceph-mon[63133]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-22T23:22:54.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:53 smithi069 ceph-mon[63133]: fsmap foofs:1/1 {0=foofs.smithi149.ickhha=up:replay} 2024-04-22T23:22:54.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 287684 9.63 MB/sec execute 584 sec latency 1350.167 ms 2024-04-22T23:22:54.450 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:53 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:22:53.992+0000 7effb28f8700 -1 bdev(0x55fce6ff8000 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 3 2024-04-22T23:22:54.450 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:22:53 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:22:53.993+0000 7effb50fd700 -1 bdev(0x55fce6ff8000 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 7 2024-04-22T23:22:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 287684 9.62 MB/sec execute 585 sec latency 2350.223 ms 2024-04-22T23:22:55.352 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:54 smithi149 ceph-mon[49797]: pgmap v299: 97 pgs: 3 remapped+peering, 9 peering, 5 active+undersized, 1 active+undersized+remapped, 18 active+undersized+degraded, 61 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.7 MiB/s wr, 329 op/s; 126/1611 objects degraded (7.821%); 2.0 MiB/s, 0 objects/s recovering 2024-04-22T23:22:55.352 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:54 smithi149 ceph-mon[49797]: osdmap e156: 8 total, 8 up, 8 in 2024-04-22T23:22:55.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:54 smithi069 ceph-mon[63133]: pgmap v299: 97 pgs: 3 remapped+peering, 9 peering, 5 active+undersized, 1 active+undersized+remapped, 18 active+undersized+degraded, 61 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.7 MiB/s wr, 329 op/s; 126/1611 objects degraded (7.821%); 2.0 MiB/s, 0 objects/s recovering 2024-04-22T23:22:55.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:54 smithi069 ceph-mon[63133]: osdmap e156: 8 total, 8 up, 8 in 2024-04-22T23:22:56.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 287684 9.60 MB/sec execute 586 sec latency 3350.284 ms 2024-04-22T23:22:56.368 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:55 smithi149 ceph-mon[49797]: osdmap e157: 8 total, 8 up, 8 in 2024-04-22T23:22:56.369 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:55 smithi149 ceph-mon[49797]: Health check update: Degraded data redundancy: 126/1611 objects degraded (7.821%), 18 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:56.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:55 smithi069 ceph-mon[63133]: osdmap e157: 8 total, 8 up, 8 in 2024-04-22T23:22:56.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:55 smithi069 ceph-mon[63133]: Health check update: Degraded data redundancy: 126/1611 objects degraded (7.821%), 18 pgs degraded (PG_DEGRADED) 2024-04-22T23:22:57.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:56 smithi149 ceph-mon[49797]: pgmap v302: 97 pgs: 1 remapped+peering, 16 peering, 1 active+undersized+remapped, 79 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 1.5 MiB/s wr, 88 op/s; 2.3 MiB/s, 28 objects/s recovering 2024-04-22T23:22:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:56 smithi149 ceph-mon[49797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 126/1611 objects degraded (7.821%), 18 pgs degraded) 2024-04-22T23:22:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:56 smithi149 ceph-mon[49797]: reconnect by client.24305 172.21.15.149:0/1186400089 after 0 2024-04-22T23:22:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:56 smithi149 ceph-mon[49797]: reconnect by client.14558 172.21.15.69:0/2880722597 after 0 2024-04-22T23:22:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:56 smithi149 ceph-mon[49797]: mds.? [v2:172.21.15.149:6832/1034958666,v1:172.21.15.149:6833/1034958666] up:reconnect 2024-04-22T23:22:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:56 smithi149 ceph-mon[49797]: fsmap foofs:1/1 {0=foofs.smithi149.ickhha=up:reconnect} 2024-04-22T23:22:57.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:22:57.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 287684 9.58 MB/sec execute 587 sec latency 4350.338 ms 2024-04-22T23:22:57.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:56 smithi069 ceph-mon[63133]: pgmap v302: 97 pgs: 1 remapped+peering, 16 peering, 1 active+undersized+remapped, 79 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 1.5 MiB/s wr, 88 op/s; 2.3 MiB/s, 28 objects/s recovering 2024-04-22T23:22:57.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:56 smithi069 ceph-mon[63133]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 126/1611 objects degraded (7.821%), 18 pgs degraded) 2024-04-22T23:22:57.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:56 smithi069 ceph-mon[63133]: reconnect by client.24305 172.21.15.149:0/1186400089 after 0 2024-04-22T23:22:57.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:56 smithi069 ceph-mon[63133]: reconnect by client.14558 172.21.15.69:0/2880722597 after 0 2024-04-22T23:22:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:56 smithi069 ceph-mon[63133]: mds.? [v2:172.21.15.149:6832/1034958666,v1:172.21.15.149:6833/1034958666] up:reconnect 2024-04-22T23:22:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:56 smithi069 ceph-mon[63133]: fsmap foofs:1/1 {0=foofs.smithi149.ickhha=up:reconnect} 2024-04-22T23:22:57.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:22:58.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:57 smithi069 ceph-mon[63133]: mds.? [v2:172.21.15.149:6832/1034958666,v1:172.21.15.149:6833/1034958666] up:rejoin 2024-04-22T23:22:58.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:57 smithi069 ceph-mon[63133]: fsmap foofs:1/1 {0=foofs.smithi149.ickhha=up:rejoin} 2024-04-22T23:22:58.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:57 smithi069 ceph-mon[63133]: daemon mds.foofs.smithi149.ickhha is now active in filesystem foofs as rank 0 2024-04-22T23:22:58.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 287769 9.58 MB/sec execute 588 sec latency 5041.582 ms 2024-04-22T23:22:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:57 smithi149 ceph-mon[49797]: mds.? [v2:172.21.15.149:6832/1034958666,v1:172.21.15.149:6833/1034958666] up:rejoin 2024-04-22T23:22:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:57 smithi149 ceph-mon[49797]: fsmap foofs:1/1 {0=foofs.smithi149.ickhha=up:rejoin} 2024-04-22T23:22:58.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:57 smithi149 ceph-mon[49797]: daemon mds.foofs.smithi149.ickhha is now active in filesystem foofs as rank 0 2024-04-22T23:22:59.008 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:58 smithi069 ceph-mon[63133]: pgmap v303: 97 pgs: 7 peering, 1 active+undersized+remapped, 89 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 964 KiB/s wr, 69 op/s; 3.5 MiB/s, 40 objects/s recovering 2024-04-22T23:22:59.009 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:58 smithi069 ceph-mon[63133]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-22T23:22:59.009 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:58 smithi069 ceph-mon[63133]: mds.? [v2:172.21.15.149:6832/1034958666,v1:172.21.15.149:6833/1034958666] up:active 2024-04-22T23:22:59.009 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:22:58 smithi069 ceph-mon[63133]: fsmap foofs:1 {0=foofs.smithi149.ickhha=up:active} 2024-04-22T23:22:59.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 288156 9.57 MB/sec execute 589 sec latency 47.992 ms 2024-04-22T23:22:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:58 smithi149 ceph-mon[49797]: pgmap v303: 97 pgs: 7 peering, 1 active+undersized+remapped, 89 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 964 KiB/s wr, 69 op/s; 3.5 MiB/s, 40 objects/s recovering 2024-04-22T23:22:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:58 smithi149 ceph-mon[49797]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-22T23:22:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:58 smithi149 ceph-mon[49797]: mds.? [v2:172.21.15.149:6832/1034958666,v1:172.21.15.149:6833/1034958666] up:active 2024-04-22T23:22:59.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:22:58 smithi149 ceph-mon[49797]: fsmap foofs:1 {0=foofs.smithi149.ickhha=up:active} 2024-04-22T23:23:00.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 288549 9.57 MB/sec execute 590 sec latency 50.321 ms 2024-04-22T23:23:01.026 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:00 smithi069 ceph-mon[63133]: pgmap v304: 97 pgs: 3 peering, 94 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 753 KiB/s wr, 62 op/s; 3.0 MiB/s, 45 objects/s recovering 2024-04-22T23:23:01.027 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:00 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:01.027 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:00 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:23:01.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 288863 9.57 MB/sec execute 591 sec latency 34.834 ms 2024-04-22T23:23:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:00 smithi149 ceph-mon[49797]: pgmap v304: 97 pgs: 3 peering, 94 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 753 KiB/s wr, 62 op/s; 3.0 MiB/s, 45 objects/s recovering 2024-04-22T23:23:01.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:00 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:01.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:00 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:23:02.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 289231 9.57 MB/sec execute 592 sec latency 59.131 ms 2024-04-22T23:23:03.049 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:02 smithi069 ceph-mon[63133]: pgmap v305: 97 pgs: 97 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 1.5 MiB/s wr, 117 op/s; 1.5 MiB/s, 38 objects/s recovering 2024-04-22T23:23:03.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 289640 9.57 MB/sec execute 593 sec latency 45.918 ms 2024-04-22T23:23:03.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:02 smithi149 ceph-mon[49797]: pgmap v305: 97 pgs: 97 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 1.5 MiB/s wr, 117 op/s; 1.5 MiB/s, 38 objects/s recovering 2024-04-22T23:23:04.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 289967 9.57 MB/sec execute 594 sec latency 120.669 ms 2024-04-22T23:23:05.083 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:05 smithi069 ceph-mon[63133]: pgmap v306: 97 pgs: 97 active+clean; 355 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 2.0 MiB/s wr, 230 op/s; 1.2 MiB/s, 31 objects/s recovering 2024-04-22T23:23:05.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 290208 9.57 MB/sec execute 595 sec latency 64.821 ms 2024-04-22T23:23:05.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:05 smithi149 ceph-mon[49797]: pgmap v306: 97 pgs: 97 active+clean; 355 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 2.0 MiB/s wr, 230 op/s; 1.2 MiB/s, 31 objects/s recovering 2024-04-22T23:23:06.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:06 smithi069 ceph-mon[63133]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-22T23:23:06.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:06 smithi069 ceph-mon[63133]: Cluster is now healthy 2024-04-22T23:23:06.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:06 smithi069 ceph-mon[63133]: mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358] up:boot 2024-04-22T23:23:06.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:06 smithi069 ceph-mon[63133]: fsmap foofs:1 {0=foofs.smithi149.ickhha=up:active} 1 up:standby 2024-04-22T23:23:06.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:06 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:23:06.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:06 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:06.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 290911 9.58 MB/sec execute 596 sec latency 36.842 ms 2024-04-22T23:23:06.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:06 smithi149 ceph-mon[49797]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-22T23:23:06.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:06 smithi149 ceph-mon[49797]: Cluster is now healthy 2024-04-22T23:23:06.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:06 smithi149 ceph-mon[49797]: mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358] up:boot 2024-04-22T23:23:06.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:06 smithi149 ceph-mon[49797]: fsmap foofs:1 {0=foofs.smithi149.ickhha=up:active} 1 up:standby 2024-04-22T23:23:06.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:06 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:23:06.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:06 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:07.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:07 smithi069 ceph-mon[63133]: pgmap v307: 97 pgs: 97 active+clean; 354 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 246 op/s; 1.0 MiB/s, 18 objects/s recovering 2024-04-22T23:23:07.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:07 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:07.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:07 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:23:07.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 291451 9.59 MB/sec execute 597 sec latency 36.748 ms 2024-04-22T23:23:07.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:07 smithi149 ceph-mon[49797]: pgmap v307: 97 pgs: 97 active+clean; 354 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 246 op/s; 1.0 MiB/s, 18 objects/s recovering 2024-04-22T23:23:07.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:07 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:07.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:07 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:23:08.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 291758 9.59 MB/sec execute 598 sec latency 63.142 ms 2024-04-22T23:23:09.137 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:09 smithi069 ceph-mon[63133]: pgmap v308: 97 pgs: 97 active+clean; 359 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.5 MiB/s wr, 354 op/s; 938 KiB/s, 16 objects/s recovering 2024-04-22T23:23:09.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 292067 9.58 MB/sec execute 599 sec latency 59.777 ms 2024-04-22T23:23:09.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:09 smithi149 ceph-mon[49797]: pgmap v308: 97 pgs: 97 active+clean; 359 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.5 MiB/s wr, 354 op/s; 938 KiB/s, 16 objects/s recovering 2024-04-22T23:23:10.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 cleanup 600 sec 2024-04-22T23:23:10.559 INFO:teuthology.orchestra.run.smithi069.stdout: 0 cleanup 600 sec 2024-04-22T23:23:10.559 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:23:10.559 INFO:teuthology.orchestra.run.smithi069.stdout: Operation Count AvgLat MaxLat 2024-04-22T23:23:10.559 INFO:teuthology.orchestra.run.smithi069.stdout: ---------------------------------------- 2024-04-22T23:23:10.559 INFO:teuthology.orchestra.run.smithi069.stdout: NTCreateX 184356 4.621 2573.491 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: Close 135471 0.134 11.237 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: Rename 7820 9.003 145.044 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: Unlink 37223 6.987 2479.902 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: Qpathinfo 167334 1.189 1719.265 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: Qfileinfo 29182 0.007 2.100 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: Qfsinfo 30637 0.628 32.996 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: Sfileinfo 15059 13.424 5030.326 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: Find 64647 0.938 17.041 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: WriteX 91179 14.385 5041.246 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: ReadX 289216 0.010 3.110 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: LockX 600 0.375 3.054 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: UnlockX 600 0.335 2.659 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: Flush 12948 0.002 0.041 2024-04-22T23:23:10.560 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:23:10.561 INFO:teuthology.orchestra.run.smithi069.stdout:Throughput 9.58273 MB/sec 5 clients 5 procs max_latency=5041.582 ms 2024-04-22T23:23:10.562 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-04-22T23:23:10.598 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-04-22T23:23:10.725 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-04-22T23:23:10.748 INFO:teuthology.orchestra.run.smithi069.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-22T23:23:10.748 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:23:10.748 INFO:teuthology.orchestra.run.smithi069.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-04-22T23:23:10.749 INFO:teuthology.orchestra.run.smithi069.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-22T23:23:10.849 INFO:teuthology.orchestra.run.smithi069.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-22T23:23:10.849 INFO:teuthology.orchestra.run.smithi069.stdout:releasing clients 2024-04-22T23:23:11.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:11 smithi069 ceph-mon[63133]: pgmap v309: 97 pgs: 97 active+clean; 351 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.9 MiB/s wr, 389 op/s; 193 KiB/s, 8 objects/s recovering 2024-04-22T23:23:11.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:11 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:11 smithi149 ceph-mon[49797]: pgmap v309: 97 pgs: 97 active+clean; 351 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.9 MiB/s wr, 389 op/s; 193 KiB/s, 8 objects/s recovering 2024-04-22T23:23:11.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:11 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:12.849 INFO:teuthology.orchestra.run.smithi069.stdout: 5 373 24.19 MB/sec execute 1 sec latency 60.638 ms 2024-04-22T23:23:13.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:13 smithi069 ceph-mon[63133]: pgmap v310: 97 pgs: 97 active+clean; 365 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 6.1 MiB/s wr, 529 op/s 2024-04-22T23:23:13.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:13 smithi149 ceph-mon[49797]: pgmap v310: 97 pgs: 97 active+clean; 365 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 6.1 MiB/s wr, 529 op/s 2024-04-22T23:23:13.849 INFO:teuthology.orchestra.run.smithi069.stdout: 5 484 21.20 MB/sec execute 2 sec latency 74.872 ms 2024-04-22T23:23:14.849 INFO:teuthology.orchestra.run.smithi069.stdout: 5 725 21.37 MB/sec execute 3 sec latency 36.930 ms 2024-04-22T23:23:15.124 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:14 smithi069 ceph-mon[63133]: pgmap v311: 97 pgs: 97 active+clean; 378 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 7.0 MiB/s wr, 611 op/s 2024-04-22T23:23:15.125 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:14 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:15.125 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:14 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:15.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:14 smithi149 ceph-mon[49797]: pgmap v311: 97 pgs: 97 active+clean; 378 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 7.0 MiB/s wr, 611 op/s 2024-04-22T23:23:15.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:14 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:15.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:14 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:15.849 INFO:teuthology.orchestra.run.smithi069.stdout: 5 995 17.04 MB/sec execute 4 sec latency 45.141 ms 2024-04-22T23:23:16.402 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:16.402 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:16.402 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:16.402 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:23:16.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:16.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:16.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:16.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:23:16.849 INFO:teuthology.orchestra.run.smithi069.stdout: 5 cleanup 5 sec 2024-04-22T23:23:17.086 INFO:teuthology.orchestra.run.smithi069.stdout: 0 cleanup 5 sec 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: Operation Count AvgLat MaxLat 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: ---------------------------------------- 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: NTCreateX 801 8.395 74.866 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: Close 733 7.815 51.646 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: Rename 50 13.296 45.135 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: Unlink 85 3.625 23.281 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: Qpathinfo 474 1.404 25.702 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: Qfileinfo 347 17.860 62.181 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: Qfsinfo 165 0.438 4.531 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: Sfileinfo 170 15.556 26.290 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: Find 232 0.663 1.804 2024-04-22T23:23:17.087 INFO:teuthology.orchestra.run.smithi069.stdout: WriteX 1632 0.025 0.165 2024-04-22T23:23:17.088 INFO:teuthology.orchestra.run.smithi069.stdout: ReadX 1112 0.018 3.006 2024-04-22T23:23:17.088 INFO:teuthology.orchestra.run.smithi069.stdout: Flush 120 15.583 25.746 2024-04-22T23:23:17.088 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:23:17.088 INFO:teuthology.orchestra.run.smithi069.stdout:Throughput 17.041 MB/sec 5 clients 5 procs max_latency=74.872 ms 2024-04-22T23:23:17.089 DEBUG:teuthology.parallel:result is None 2024-04-22T23:23:17.261 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:23:17.615 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:17 smithi069 ceph-mon[63133]: pgmap v312: 97 pgs: 97 active+clean; 368 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 11 MiB/s wr, 555 op/s 2024-04-22T23:23:17.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:17 smithi149 ceph-mon[49797]: pgmap v312: 97 pgs: 97 active+clean; 368 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 11 MiB/s wr, 555 op/s 2024-04-22T23:23:17.955 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:23:17.955 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (9m) 4s ago 18m 25.9M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:23:17.955 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (6m) 4s ago 18m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:23:17.955 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (6m) 37s ago 18m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:23:17.955 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (9m) 4s ago 18m 54.4M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:23:17.955 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (12s) 4s ago 12m 12.5M - 19.0.0-3252-g43be0201 a0434699c1d7 f04c8fddac95 2024-04-22T23:23:17.955 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (12m) 37s ago 12m 15.6M - 17.2.0 e1d6a67b021e 00c3a8b84ec6 2024-04-22T23:23:17.955 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (8m) 4s ago 19m 514M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:23:17.955 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (8m) 37s ago 17m 453M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:23:17.955 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (7m) 4s ago 20m 60.3M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:23:17.955 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (7m) 37s ago 17m 42.6M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:23:17.955 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (12m) 4s ago 12m 282M - 4.0 e1d6a67b021e 6135fc86685d 2024-04-22T23:23:17.956 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (12m) 37s ago 12m 86.9M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:23:17.956 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (9m) 4s ago 18m 20.9M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:23:17.956 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (9m) 37s ago 17m 19.2M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:23:17.956 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (5m) 4s ago 17m 429M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:23:17.956 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (4m) 4s ago 16m 392M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:23:17.956 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (3m) 4s ago 16m 300M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:23:17.956 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (3m) 4s ago 15m 338M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 e7772e3632e1 2024-04-22T23:23:17.956 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (2m) 37s ago 15m 334M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 036a1a8ba3c9 2024-04-22T23:23:17.956 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (118s) 37s ago 14m 284M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 539c90bd7194 2024-04-22T23:23:17.956 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (80s) 37s ago 14m 278M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 b821ba1ee812 2024-04-22T23:23:17.956 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (42s) 37s ago 14m 15.1M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 fec23dae1ff2 2024-04-22T23:23:17.956 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (8m) 4s ago 17m 76.2M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:23:18.416 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:23:18.416 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:23:18.416 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:23:18.416 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:23:18.416 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:23:18.416 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:23:18.416 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:23:18.416 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:23:18.417 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 8 2024-04-22T23:23:18.417 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:23:18.417 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:23:18.417 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T23:23:18.417 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 1 2024-04-22T23:23:18.417 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:23:18.417 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:23:18.417 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T23:23:18.417 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 13 2024-04-22T23:23:18.417 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:23:18.417 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:23:18.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:18 smithi069 ceph-mon[63133]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:18.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:18 smithi149 ceph-mon[49797]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:18.788 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:23:18.788 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:23:18.789 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:23:18.789 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:23:18.789 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:23:18.789 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:23:18.789 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:23:18.789 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-04-22T23:23:18.789 INFO:teuthology.orchestra.run.smithi069.stdout: "osd" 2024-04-22T23:23:18.789 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:23:18.789 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "15/23 daemons upgraded", 2024-04-22T23:23:18.789 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading mds daemons", 2024-04-22T23:23:18.789 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:23:18.789 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:23:19.333 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:23:19.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:19 smithi069 ceph-mon[63133]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:19.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:19 smithi069 ceph-mon[63133]: pgmap v313: 97 pgs: 97 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 12 MiB/s wr, 548 op/s 2024-04-22T23:23:19.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:19 smithi069 ceph-mon[63133]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:19.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:19 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/626694364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:19.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:19 smithi149 ceph-mon[49797]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:19.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:19 smithi149 ceph-mon[49797]: pgmap v313: 97 pgs: 97 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 12 MiB/s wr, 548 op/s 2024-04-22T23:23:19.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:19 smithi149 ceph-mon[49797]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:19.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:19 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/626694364' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:20.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:20.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/332691903' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:23:20.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:20.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:20.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:20.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:23:20.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:20.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:20.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:23:20.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:20.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:20.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:20.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:20.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi149.ickhha"]}]: dispatch 2024-04-22T23:23:20.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:20.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/332691903' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:23:20.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:20.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:20.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:20.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:23:20.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:20.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:20.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:23:20.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:20.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:20.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:20.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:20.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi149.ickhha"]}]: dispatch 2024-04-22T23:23:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:21 smithi149 ceph-mon[49797]: pgmap v314: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 12 MiB/s wr, 523 op/s 2024-04-22T23:23:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:21 smithi149 ceph-mon[49797]: Detected new or changed devices on smithi069 2024-04-22T23:23:21.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:21 smithi149 ceph-mon[49797]: Upgrade: It appears safe to stop mds.foofs.smithi149.ickhha 2024-04-22T23:23:21.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:21 smithi069 ceph-mon[63133]: pgmap v314: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 12 MiB/s wr, 523 op/s 2024-04-22T23:23:21.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:21 smithi069 ceph-mon[63133]: Detected new or changed devices on smithi069 2024-04-22T23:23:21.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:21 smithi069 ceph-mon[63133]: Upgrade: It appears safe to stop mds.foofs.smithi149.ickhha 2024-04-22T23:23:23.180 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:22 smithi149 ceph-mon[49797]: pgmap v315: 97 pgs: 97 active+clean; 293 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 12 MiB/s wr, 504 op/s 2024-04-22T23:23:23.180 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:22 smithi149 ceph-mon[49797]: Upgrade: Updating mds.foofs.smithi149.ickhha 2024-04-22T23:23:23.180 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:22 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:23.180 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:22 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.ickhha", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T23:23:23.180 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:22 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:23.180 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:22 smithi149 ceph-mon[49797]: Deploying daemon mds.foofs.smithi149.ickhha on smithi149 2024-04-22T23:23:23.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:22 smithi069 ceph-mon[63133]: pgmap v315: 97 pgs: 97 active+clean; 293 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 12 MiB/s wr, 504 op/s 2024-04-22T23:23:23.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:22 smithi069 ceph-mon[63133]: Upgrade: Updating mds.foofs.smithi149.ickhha 2024-04-22T23:23:23.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:22 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:23.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:22 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi149.ickhha", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-22T23:23:23.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:22 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:23.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:22 smithi069 ceph-mon[63133]: Deploying daemon mds.foofs.smithi149.ickhha on smithi149 2024-04-22T23:23:24.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069[63129]: 2024-04-22T23:23:24.084+0000 7f4e9d8c7700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T23:23:24.396 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[71832]: 2024-04-22T23:23:24.106+0000 7ffa1bcc3700 -1 bdev(0x5622a0b66000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 1 2024-04-22T23:23:24.396 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[71832]: 2024-04-22T23:23:24.106+0000 7ffa18cbd700 -1 bdev(0x5622a0b66000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 1 2024-04-22T23:23:24.396 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[71832]: 2024-04-22T23:23:24.106+0000 7ffa184bc700 -1 bdev(0x5622a0b66000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 1 2024-04-22T23:23:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:24 smithi149 ceph-mon[49797]: pgmap v316: 97 pgs: 97 active+clean; 272 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 365 op/s 2024-04-22T23:23:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:24 smithi149 ceph-mon[49797]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-22T23:23:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:24 smithi149 ceph-mon[49797]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T23:23:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:24 smithi149 ceph-mon[49797]: osdmap e158: 8 total, 8 up, 8 in 2024-04-22T23:23:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:24 smithi149 ceph-mon[49797]: Standby daemon mds.foofs.smithi069.jiyres assigned to filesystem foofs as rank 0 2024-04-22T23:23:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:24 smithi149 ceph-mon[49797]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-22T23:23:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:24 smithi149 ceph-mon[49797]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-22T23:23:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:24 smithi149 ceph-mon[49797]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-22T23:23:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:24 smithi149 ceph-mon[49797]: fsmap foofs:1/1 {0=foofs.smithi069.jiyres=up:replay} 2024-04-22T23:23:25.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-mon[63133]: pgmap v316: 97 pgs: 97 active+clean; 272 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 365 op/s 2024-04-22T23:23:25.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-mon[63133]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-22T23:23:25.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-mon[63133]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-22T23:23:25.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-mon[63133]: osdmap e158: 8 total, 8 up, 8 in 2024-04-22T23:23:25.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-mon[63133]: Standby daemon mds.foofs.smithi069.jiyres assigned to filesystem foofs as rank 0 2024-04-22T23:23:25.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-mon[63133]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-22T23:23:25.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-mon[63133]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-22T23:23:25.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-mon[63133]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-22T23:23:25.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:24 smithi069 ceph-mon[63133]: fsmap foofs:1/1 {0=foofs.smithi069.jiyres=up:replay} 2024-04-22T23:23:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:26 smithi149 ceph-mon[49797]: pgmap v318: 97 pgs: 97 active+clean; 242 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.8 MiB/s wr, 220 op/s 2024-04-22T23:23:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:26 smithi149 ceph-mon[49797]: reconnect by client.14558 172.21.15.69:0/2880722597 after 0.000999979 2024-04-22T23:23:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:26 smithi149 ceph-mon[49797]: reconnect by client.24305 172.21.15.149:0/1186400089 after 0.000999979 2024-04-22T23:23:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:26 smithi149 ceph-mon[49797]: mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358] up:reconnect 2024-04-22T23:23:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:26 smithi149 ceph-mon[49797]: fsmap foofs:1/1 {0=foofs.smithi069.jiyres=up:reconnect} 2024-04-22T23:23:27.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:26 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:23:27.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:26 smithi069 ceph-mon[63133]: pgmap v318: 97 pgs: 97 active+clean; 242 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.8 MiB/s wr, 220 op/s 2024-04-22T23:23:27.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:26 smithi069 ceph-mon[63133]: reconnect by client.14558 172.21.15.69:0/2880722597 after 0.000999979 2024-04-22T23:23:27.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:26 smithi069 ceph-mon[63133]: reconnect by client.24305 172.21.15.149:0/1186400089 after 0.000999979 2024-04-22T23:23:27.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:26 smithi069 ceph-mon[63133]: mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358] up:reconnect 2024-04-22T23:23:27.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:26 smithi069 ceph-mon[63133]: fsmap foofs:1/1 {0=foofs.smithi069.jiyres=up:reconnect} 2024-04-22T23:23:27.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:26 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:23:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:27 smithi149 ceph-mon[49797]: mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358] up:rejoin 2024-04-22T23:23:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:27 smithi149 ceph-mon[49797]: fsmap foofs:1/1 {0=foofs.smithi069.jiyres=up:rejoin} 2024-04-22T23:23:28.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:27 smithi149 ceph-mon[49797]: daemon mds.foofs.smithi069.jiyres is now active in filesystem foofs as rank 0 2024-04-22T23:23:28.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:27 smithi069 ceph-mon[63133]: mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358] up:rejoin 2024-04-22T23:23:28.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:27 smithi069 ceph-mon[63133]: fsmap foofs:1/1 {0=foofs.smithi069.jiyres=up:rejoin} 2024-04-22T23:23:28.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:27 smithi069 ceph-mon[63133]: daemon mds.foofs.smithi069.jiyres is now active in filesystem foofs as rank 0 2024-04-22T23:23:29.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:28 smithi149 ceph-mon[49797]: pgmap v319: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.4 MiB/s wr, 177 op/s 2024-04-22T23:23:29.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:28 smithi149 ceph-mon[49797]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-22T23:23:29.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:28 smithi149 ceph-mon[49797]: mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358] up:active 2024-04-22T23:23:29.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:28 smithi149 ceph-mon[49797]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 2024-04-22T23:23:29.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:28 smithi069 ceph-mon[63133]: pgmap v319: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.4 MiB/s wr, 177 op/s 2024-04-22T23:23:29.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:28 smithi069 ceph-mon[63133]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-22T23:23:29.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:28 smithi069 ceph-mon[63133]: mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358] up:active 2024-04-22T23:23:29.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:28 smithi069 ceph-mon[63133]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 2024-04-22T23:23:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:30 smithi149 ceph-mon[49797]: pgmap v320: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 460 KiB/s wr, 55 op/s 2024-04-22T23:23:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:23:31.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:30 smithi069 ceph-mon[63133]: pgmap v320: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 460 KiB/s wr, 55 op/s 2024-04-22T23:23:31.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:31.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:23:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:32 smithi149 ceph-mon[49797]: pgmap v321: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 52 KiB/s wr, 25 op/s 2024-04-22T23:23:33.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:32 smithi069 ceph-mon[63133]: pgmap v321: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 52 KiB/s wr, 25 op/s 2024-04-22T23:23:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:34 smithi149 ceph-mon[49797]: pgmap v322: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 22 KiB/s wr, 23 op/s 2024-04-22T23:23:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:34 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:34 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:35.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:34 smithi069 ceph-mon[63133]: pgmap v322: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 22 KiB/s wr, 23 op/s 2024-04-22T23:23:35.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:34 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:35.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:34 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:36.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:35 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:23:36.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:35 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:23:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:36 smithi149 ceph-mon[49797]: pgmap v323: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 19 KiB/s wr, 19 op/s 2024-04-22T23:23:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:36 smithi149 ceph-mon[49797]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-22T23:23:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:36 smithi149 ceph-mon[49797]: Cluster is now healthy 2024-04-22T23:23:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:36 smithi149 ceph-mon[49797]: mds.? [v2:172.21.15.149:6832/1387420041,v1:172.21.15.149:6833/1387420041] up:boot 2024-04-22T23:23:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:36 smithi149 ceph-mon[49797]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 1 up:standby 2024-04-22T23:23:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:36 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:23:37.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:36 smithi069 ceph-mon[63133]: pgmap v323: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 19 KiB/s wr, 19 op/s 2024-04-22T23:23:37.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:36 smithi069 ceph-mon[63133]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-22T23:23:37.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:36 smithi069 ceph-mon[63133]: Cluster is now healthy 2024-04-22T23:23:37.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:36 smithi069 ceph-mon[63133]: mds.? [v2:172.21.15.149:6832/1387420041,v1:172.21.15.149:6833/1387420041] up:boot 2024-04-22T23:23:37.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:36 smithi069 ceph-mon[63133]: fsmap foofs:1 {0=foofs.smithi069.jiyres=up:active} 1 up:standby 2024-04-22T23:23:37.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:36 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:23:39.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:38 smithi069 ceph-mon[63133]: pgmap v324: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 18 KiB/s wr, 13 op/s 2024-04-22T23:23:39.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:39.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:38 smithi149 ceph-mon[49797]: pgmap v324: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 18 KiB/s wr, 13 op/s 2024-04-22T23:23:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:39.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:40.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:40 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:40.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:40 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:40.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:40 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:40.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:40 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:41 smithi149 ceph-mon[49797]: pgmap v325: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 18 KiB/s wr, 9 op/s 2024-04-22T23:23:41.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:41 smithi069 ceph-mon[63133]: pgmap v325: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 18 KiB/s wr, 9 op/s 2024-04-22T23:23:43.415 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: pgmap v326: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 7.1 KiB/s wr, 5 op/s 2024-04-22T23:23:43.415 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.415 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi069.jiyres"}]': finished 2024-04-22T23:23:43.416 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:23:43.417 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi149.ickhha"}]': finished 2024-04-22T23:23:43.417 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.417 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.417 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.417 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.417 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.417 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.417 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.417 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.417 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: pgmap v326: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 7.1 KiB/s wr, 5 op/s 2024-04-22T23:23:43.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi069.jiyres"}]: dispatch 2024-04-22T23:23:43.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi069.jiyres"}]': finished 2024-04-22T23:23:43.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi149.ickhha"}]: dispatch 2024-04-22T23:23:43.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi149.ickhha"}]': finished 2024-04-22T23:23:43.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:43.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:43.452 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:44.438 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:44 smithi069 ceph-mon[63133]: Detected new or changed devices on smithi149 2024-04-22T23:23:44.438 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:44 smithi069 ceph-mon[63133]: Upgrade: Setting container_image for all mds 2024-04-22T23:23:44.438 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:44 smithi069 ceph-mon[63133]: Upgrade: Setting container_image for all rgw 2024-04-22T23:23:44.438 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:44 smithi069 ceph-mon[63133]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T23:23:44.438 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:44 smithi069 ceph-mon[63133]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T23:23:44.438 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:44 smithi069 ceph-mon[63133]: Upgrade: Setting container_image for all iscsi 2024-04-22T23:23:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:44 smithi149 ceph-mon[49797]: Detected new or changed devices on smithi149 2024-04-22T23:23:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:44 smithi149 ceph-mon[49797]: Upgrade: Setting container_image for all mds 2024-04-22T23:23:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:44 smithi149 ceph-mon[49797]: Upgrade: Setting container_image for all rgw 2024-04-22T23:23:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:44 smithi149 ceph-mon[49797]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T23:23:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:44 smithi149 ceph-mon[49797]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T23:23:44.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:44 smithi149 ceph-mon[49797]: Upgrade: Setting container_image for all iscsi 2024-04-22T23:23:45.386 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:45 smithi069 ceph-mon[63133]: pgmap v327: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-04-22T23:23:45.386 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:45.386 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.cdmddj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T23:23:45.386 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T23:23:45.386 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T23:23:45.386 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:45.386 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T23:23:45.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T23:23:45.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.cdmddj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T23:23:45.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:45 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:45 smithi149 ceph-mon[49797]: pgmap v327: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-04-22T23:23:45.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.cdmddj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T23:23:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T23:23:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T23:23:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T23:23:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T23:23:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.cdmddj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T23:23:45.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:45 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:46.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:46 smithi149 ceph-mon[49797]: Upgrade: Updating nfs.foo.0.0.smithi069.cdmddj (1/2) 2024-04-22T23:23:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:46 smithi149 ceph-mon[49797]: Creating key for client.nfs.foo.0.0.smithi069.cdmddj 2024-04-22T23:23:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:46 smithi149 ceph-mon[49797]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-22T23:23:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:46 smithi149 ceph-mon[49797]: Rados config object exists: conf-nfs.foo 2024-04-22T23:23:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:46 smithi149 ceph-mon[49797]: Creating key for client.nfs.foo.0.0.smithi069.cdmddj-rgw 2024-04-22T23:23:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:46 smithi149 ceph-mon[49797]: Bind address in nfs.foo.0.0.smithi069.cdmddj's ganesha conf is defaulting to empty 2024-04-22T23:23:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:46 smithi149 ceph-mon[49797]: Deploying daemon nfs.foo.0.0.smithi069.cdmddj on smithi069 2024-04-22T23:23:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:46.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:23:46.510 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:46 smithi069 ceph-mon[63133]: Upgrade: Updating nfs.foo.0.0.smithi069.cdmddj (1/2) 2024-04-22T23:23:46.510 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:46 smithi069 ceph-mon[63133]: Creating key for client.nfs.foo.0.0.smithi069.cdmddj 2024-04-22T23:23:46.510 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:46 smithi069 ceph-mon[63133]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-22T23:23:46.510 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:46 smithi069 ceph-mon[63133]: Rados config object exists: conf-nfs.foo 2024-04-22T23:23:46.510 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:46 smithi069 ceph-mon[63133]: Creating key for client.nfs.foo.0.0.smithi069.cdmddj-rgw 2024-04-22T23:23:46.510 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:46 smithi069 ceph-mon[63133]: Bind address in nfs.foo.0.0.smithi069.cdmddj's ganesha conf is defaulting to empty 2024-04-22T23:23:46.510 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:46 smithi069 ceph-mon[63133]: Deploying daemon nfs.foo.0.0.smithi069.cdmddj on smithi069 2024-04-22T23:23:46.510 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:46.511 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:23:47.397 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:47 smithi069 ceph-mon[63133]: pgmap v328: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-04-22T23:23:47.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:47 smithi149 ceph-mon[49797]: pgmap v328: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-04-22T23:23:49.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:49 smithi149 ceph-mon[49797]: pgmap v329: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 170 B/s wr, 0 op/s 2024-04-22T23:23:49.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:49 smithi149 ceph-mon[49797]: Evicting (and blocklisting) client session 14558 (172.21.15.69:0/2880722597) 2024-04-22T23:23:49.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:49 smithi149 ceph-mon[49797]: from='mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358]' entity='mds.foofs.smithi069.jiyres' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.69:0/2880722597"}]: dispatch 2024-04-22T23:23:49.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:49.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:49 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:49.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:49 smithi069 ceph-mon[63133]: pgmap v329: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 170 B/s wr, 0 op/s 2024-04-22T23:23:49.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:49 smithi069 ceph-mon[63133]: Evicting (and blocklisting) client session 14558 (172.21.15.69:0/2880722597) 2024-04-22T23:23:49.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:49 smithi069 ceph-mon[63133]: from='mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358]' entity='mds.foofs.smithi069.jiyres' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.69:0/2880722597"}]: dispatch 2024-04-22T23:23:49.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:49.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:49 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:49.694 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:23:50.383 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:23:50.383 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (10m) 36s ago 19m 25.9M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:23:50.383 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (6m) 36s ago 19m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:23:50.383 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (6m) 12s ago 18m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:23:50.383 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (9m) 36s ago 18m 54.4M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:23:50.383 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (45s) 36s ago 13m 12.5M - 19.0.0-3252-g43be0201 a0434699c1d7 f04c8fddac95 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (15s) 12s ago 13m 13.7M - 19.0.0-3252-g43be0201 a0434699c1d7 a44cb1d5a336 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (9m) 36s ago 20m 514M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (8m) 12s ago 18m 453M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (8m) 36s ago 20m 60.3M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (7m) 12s ago 18m 46.4M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 starting - - - - 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (12m) 12s ago 12m 87.0M - 4.0 e1d6a67b021e 45f95a8c5b8d 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (9m) 36s ago 18m 20.9M - 1.3.1 1dbe0e931976 12d1407ca4b3 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (9m) 12s ago 18m 19.0M - 1.3.1 1dbe0e931976 a0db63f4b91f 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (6m) 36s ago 17m 429M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (5m) 36s ago 17m 392M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (4m) 36s ago 16m 300M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:23:50.384 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (3m) 36s ago 16m 338M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 e7772e3632e1 2024-04-22T23:23:50.385 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (3m) 12s ago 15m 376M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 036a1a8ba3c9 2024-04-22T23:23:50.385 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (2m) 12s ago 15m 340M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 539c90bd7194 2024-04-22T23:23:50.385 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (112s) 12s ago 15m 347M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 b821ba1ee812 2024-04-22T23:23:50.385 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (74s) 12s ago 14m 335M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 fec23dae1ff2 2024-04-22T23:23:50.385 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (8m) 36s ago 18m 76.2M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:23:50.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:50 smithi149 ceph-mon[49797]: from='mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358]' entity='mds.foofs.smithi069.jiyres' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.69:0/2880722597"}]': finished 2024-04-22T23:23:50.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:50 smithi149 ceph-mon[49797]: osdmap e159: 8 total, 8 up, 8 in 2024-04-22T23:23:50.598 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:50 smithi069 ceph-mon[63133]: from='mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358]' entity='mds.foofs.smithi069.jiyres' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.69:0/2880722597"}]': finished 2024-04-22T23:23:50.598 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:50 smithi069 ceph-mon[63133]: osdmap e159: 8 total, 8 up, 8 in 2024-04-22T23:23:50.807 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:23:50.807 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:23:50.807 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:23:50.807 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:23:50.807 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:23:50.807 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:23:50.808 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:23:50.808 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:23:50.808 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 8 2024-04-22T23:23:50.808 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:23:50.808 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:23:50.808 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:23:50.808 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:23:50.808 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:23:50.808 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 14 2024-04-22T23:23:50.808 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:23:50.808 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:23:51.169 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:23:51.170 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:23:51.170 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:23:51.170 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:23:51.170 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:23:51.170 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:23:51.170 INFO:teuthology.orchestra.run.smithi069.stdout: "osd", 2024-04-22T23:23:51.170 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:23:51.170 INFO:teuthology.orchestra.run.smithi069.stdout: "mds", 2024-04-22T23:23:51.170 INFO:teuthology.orchestra.run.smithi069.stdout: "mon" 2024-04-22T23:23:51.170 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:23:51.170 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "16/23 daemons upgraded", 2024-04-22T23:23:51.170 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading nfs daemons", 2024-04-22T23:23:51.171 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:23:51.171 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:23:51.443 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='client.34518 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:51.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: pgmap v331: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 204 B/s wr, 0 op/s 2024-04-22T23:23:51.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='client.34522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:51.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='client.34526 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:51.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:51.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.mxpcoc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T23:23:51.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T23:23:51.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T23:23:51.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:51.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T23:23:51.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T23:23:51.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.mxpcoc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T23:23:51.444 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:51.445 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:51 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/1251192543' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:51.588 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:23:51.589 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='client.34518 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:51.589 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: pgmap v331: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 204 B/s wr, 0 op/s 2024-04-22T23:23:51.589 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='client.34522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:51.589 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='client.34526 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:51.589 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:51.589 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.mxpcoc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-22T23:23:51.589 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-22T23:23:51.589 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-22T23:23:51.589 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:51.589 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-22T23:23:51.590 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-22T23:23:51.590 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi149.mxpcoc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T23:23:51.590 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:23:51.590 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:51 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/1251192543' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:23:52.377 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:52 smithi149 ceph-mon[49797]: Upgrade: Updating nfs.foo.1.0.smithi149.mxpcoc (2/2) 2024-04-22T23:23:52.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:52 smithi149 ceph-mon[49797]: Creating key for client.nfs.foo.1.0.smithi149.mxpcoc 2024-04-22T23:23:52.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:52 smithi149 ceph-mon[49797]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-22T23:23:52.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:52 smithi149 ceph-mon[49797]: Rados config object exists: conf-nfs.foo 2024-04-22T23:23:52.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:52 smithi149 ceph-mon[49797]: Creating key for client.nfs.foo.1.0.smithi149.mxpcoc-rgw 2024-04-22T23:23:52.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:52 smithi149 ceph-mon[49797]: Bind address in nfs.foo.1.0.smithi149.mxpcoc's ganesha conf is defaulting to empty 2024-04-22T23:23:52.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:52 smithi149 ceph-mon[49797]: Deploying daemon nfs.foo.1.0.smithi149.mxpcoc on smithi149 2024-04-22T23:23:52.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:52 smithi149 ceph-mon[49797]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:52.378 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:52 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/3475403105' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:23:52.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:52 smithi069 ceph-mon[63133]: Upgrade: Updating nfs.foo.1.0.smithi149.mxpcoc (2/2) 2024-04-22T23:23:52.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:52 smithi069 ceph-mon[63133]: Creating key for client.nfs.foo.1.0.smithi149.mxpcoc 2024-04-22T23:23:52.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:52 smithi069 ceph-mon[63133]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-22T23:23:52.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:52 smithi069 ceph-mon[63133]: Rados config object exists: conf-nfs.foo 2024-04-22T23:23:52.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:52 smithi069 ceph-mon[63133]: Creating key for client.nfs.foo.1.0.smithi149.mxpcoc-rgw 2024-04-22T23:23:52.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:52 smithi069 ceph-mon[63133]: Bind address in nfs.foo.1.0.smithi149.mxpcoc's ganesha conf is defaulting to empty 2024-04-22T23:23:52.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:52 smithi069 ceph-mon[63133]: Deploying daemon nfs.foo.1.0.smithi149.mxpcoc on smithi149 2024-04-22T23:23:52.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:52 smithi069 ceph-mon[63133]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:23:52.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:52 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/3475403105' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:23:53.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:53 smithi149 ceph-mon[49797]: pgmap v332: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 819 B/s wr, 3 op/s 2024-04-22T23:23:53.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:53 smithi069 ceph-mon[63133]: pgmap v332: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 819 B/s wr, 3 op/s 2024-04-22T23:23:55.512 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:55 smithi149 ceph-mon[49797]: pgmap v333: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.1 KiB/s wr, 4 op/s 2024-04-22T23:23:55.513 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:55 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:55.513 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:55 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:55.513 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:55 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:23:55.520 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:55 smithi069 ceph-mon[63133]: pgmap v333: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.1 KiB/s wr, 4 op/s 2024-04-22T23:23:55.520 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:55 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:55.520 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:55 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:55.520 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:55 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:23:57.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:57 smithi149 ceph-mon[49797]: pgmap v334: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.1 KiB/s wr, 4 op/s 2024-04-22T23:23:57.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:57 smithi069 ceph-mon[63133]: pgmap v334: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.1 KiB/s wr, 4 op/s 2024-04-22T23:23:58.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:58 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:58.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:58 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:58.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:58 smithi069 ceph-mon[63133]: pgmap v335: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-04-22T23:23:58.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:58 smithi069 ceph-mon[63133]: from='mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358]' entity='mds.foofs.smithi069.jiyres' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.149:0/1186400089"}]: dispatch 2024-04-22T23:23:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:58 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:58 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:58 smithi149 ceph-mon[49797]: pgmap v335: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-04-22T23:23:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:58 smithi149 ceph-mon[49797]: from='mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358]' entity='mds.foofs.smithi069.jiyres' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.149:0/1186400089"}]: dispatch 2024-04-22T23:23:59.804 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:59 smithi149 ceph-mon[49797]: Evicting (and blocklisting) client session 24305 (172.21.15.149:0/1186400089) 2024-04-22T23:23:59.804 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:59 smithi149 ceph-mon[49797]: from='mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358]' entity='mds.foofs.smithi069.jiyres' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.149:0/1186400089"}]': finished 2024-04-22T23:23:59.804 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:59 smithi149 ceph-mon[49797]: osdmap e160: 8 total, 8 up, 8 in 2024-04-22T23:23:59.804 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:59 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:23:59.804 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:23:59 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:00.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:59 smithi069 ceph-mon[63133]: Evicting (and blocklisting) client session 24305 (172.21.15.149:0/1186400089) 2024-04-22T23:24:00.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:59 smithi069 ceph-mon[63133]: from='mds.? [v2:172.21.15.69:6834/3101370358,v1:172.21.15.69:6835/3101370358]' entity='mds.foofs.smithi069.jiyres' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.149:0/1186400089"}]': finished 2024-04-22T23:24:00.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:59 smithi069 ceph-mon[63133]: osdmap e160: 8 total, 8 up, 8 in 2024-04-22T23:24:00.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:59 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:00.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:23:59 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:00.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:00 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:00.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:00 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:00.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:00 smithi069 ceph-mon[63133]: pgmap v337: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-04-22T23:24:00.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:00 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:00.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:00 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:00.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:00 smithi149 ceph-mon[49797]: pgmap v337: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-04-22T23:24:01.826 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:24:01.826 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:01.826 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:01.889 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:24:01.889 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:01.889 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:03.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:03 smithi069 ceph-mon[63133]: pgmap v338: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-04-22T23:24:03.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:03 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:03.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:03 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:03.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:03 smithi149 ceph-mon[49797]: pgmap v338: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-04-22T23:24:03.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:03 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:03.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:03 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:05.524 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: pgmap v339: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-22T23:24:05.524 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:05.524 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:05.524 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:24:05.524 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:24:05.524 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:05.524 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:05.525 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi069.cdmddj"}]: dispatch 2024-04-22T23:24:05.526 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi069.cdmddj"}]': finished 2024-04-22T23:24:05.526 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi149.mxpcoc"}]: dispatch 2024-04-22T23:24:05.526 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi149.mxpcoc"}]': finished 2024-04-22T23:24:05.526 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.526 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:05 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:05.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: pgmap v339: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:05.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi069.cdmddj"}]: dispatch 2024-04-22T23:24:05.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi069.cdmddj"}]': finished 2024-04-22T23:24:05.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi149.mxpcoc"}]: dispatch 2024-04-22T23:24:05.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi149.mxpcoc"}]': finished 2024-04-22T23:24:05.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:05.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:05 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:06.526 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:06 smithi069 ceph-mon[63133]: Upgrade: Setting container_image for all nfs 2024-04-22T23:24:06.527 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:06 smithi069 ceph-mon[63133]: Upgrade: Setting container_image for all nvmeof 2024-04-22T23:24:06.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:06 smithi149 ceph-mon[49797]: Upgrade: Setting container_image for all nfs 2024-04-22T23:24:06.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:06 smithi149 ceph-mon[49797]: Upgrade: Setting container_image for all nvmeof 2024-04-22T23:24:07.486 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:07 smithi069 ceph-mon[63133]: pgmap v340: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-22T23:24:07.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:07 smithi149 ceph-mon[49797]: pgmap v340: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-22T23:24:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:08 smithi069 ceph-mon[63133]: Upgrade: Updating node-exporter.smithi069 (1/2) 2024-04-22T23:24:08.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:08 smithi069 ceph-mon[63133]: Deploying daemon node-exporter.smithi069 on smithi069 2024-04-22T23:24:08.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:08 smithi149 ceph-mon[49797]: Upgrade: Updating node-exporter.smithi069 (1/2) 2024-04-22T23:24:08.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:08 smithi149 ceph-mon[49797]: Deploying daemon node-exporter.smithi069 on smithi069 2024-04-22T23:24:09.540 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:09 smithi069 ceph-mon[63133]: pgmap v341: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 409 B/s wr, 0 op/s 2024-04-22T23:24:09.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:09 smithi149 ceph-mon[49797]: pgmap v341: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 409 B/s wr, 0 op/s 2024-04-22T23:24:11.548 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:11 smithi069 ceph-mon[63133]: pgmap v342: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 B/s rd, 275 B/s wr, 0 op/s 2024-04-22T23:24:11.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:11 smithi149 ceph-mon[49797]: pgmap v342: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 B/s rd, 275 B/s wr, 0 op/s 2024-04-22T23:24:13.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:12 smithi149 ceph-mon[49797]: pgmap v343: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-04-22T23:24:13.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:12 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:13.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:12 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:13.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:12 smithi069 ceph-mon[63133]: pgmap v343: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-04-22T23:24:13.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:12 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:13.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:12 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:15.150 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:14 smithi149 ceph-mon[49797]: Upgrade: Updating node-exporter.smithi149 (2/2) 2024-04-22T23:24:15.151 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:14 smithi149 ceph-mon[49797]: Deploying daemon node-exporter.smithi149 on smithi149 2024-04-22T23:24:15.151 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:14 smithi149 ceph-mon[49797]: pgmap v344: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-04-22T23:24:15.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:14 smithi069 ceph-mon[63133]: Upgrade: Updating node-exporter.smithi149 (2/2) 2024-04-22T23:24:15.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:14 smithi069 ceph-mon[63133]: Deploying daemon node-exporter.smithi149 on smithi149 2024-04-22T23:24:15.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:14 smithi069 ceph-mon[63133]: pgmap v344: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-04-22T23:24:16.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:15 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:24:16.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:15 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:24:17.311 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:16 smithi149 ceph-mon[49797]: pgmap v345: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:17.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:16 smithi069 ceph-mon[63133]: pgmap v345: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:19.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:18 smithi149 ceph-mon[49797]: pgmap v346: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:19.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:18 smithi069 ceph-mon[63133]: pgmap v346: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:20.863 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:20.863 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:20.863 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:20 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:24:20.863 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:20 smithi069 ceph-mon[63133]: pgmap v347: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:20.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:20.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:20.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:20 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:24:20.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:20 smithi149 ceph-mon[49797]: pgmap v347: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:21.958 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:24:22.653 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:24:22.653 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (10m) 22s ago 19m 25.9M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:24:22.653 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (7m) 22s ago 19m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (7m) 25s ago 19m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (10m) 22s ago 19m 54.0M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (77s) 22s ago 13m 35.3M - 19.0.0-3252-g43be0201 a0434699c1d7 f04c8fddac95 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (48s) 25s ago 13m 15.0M - 19.0.0-3252-g43be0201 a0434699c1d7 a44cb1d5a336 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (10m) 22s ago 20m 515M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (9m) 25s ago 19m 453M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (8m) 22s ago 21m 60.7M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (8m) 25s ago 18m 47.0M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (34s) 22s ago 13m 68.6M - 5.7 a0434699c1d7 c41386c372b1 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (28s) 25s ago 13m 18.4M - 5.7 a0434699c1d7 9f0fcbc0fa9a 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 starting - - - - 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 starting - - - - 2024-04-22T23:24:22.654 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (6m) 22s ago 18m 435M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:24:22.655 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (5m) 22s ago 17m 397M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:24:22.655 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (5m) 22s ago 17m 307M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:24:22.655 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (4m) 22s ago 16m 354M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 e7772e3632e1 2024-04-22T23:24:22.655 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (3m) 25s ago 16m 376M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 036a1a8ba3c9 2024-04-22T23:24:22.655 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (3m) 25s ago 16m 340M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 539c90bd7194 2024-04-22T23:24:22.655 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (2m) 25s ago 15m 347M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 b821ba1ee812 2024-04-22T23:24:22.655 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (107s) 25s ago 15m 335M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 fec23dae1ff2 2024-04-22T23:24:22.655 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (9m) 22s ago 18m 76.7M - 2.33.4 514e6a882f6e 18a35cb61461 2024-04-22T23:24:23.088 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 8 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:24:23.089 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:24:23.090 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 14 2024-04-22T23:24:23.090 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:24:23.090 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:24:23.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:22 smithi069 ceph-mon[63133]: pgmap v348: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:24:23.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:22 smithi069 ceph-mon[63133]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:23.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:22 smithi069 ceph-mon[63133]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:23.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:22 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:23.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:22 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:23.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:22 smithi149 ceph-mon[49797]: pgmap v348: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:24:23.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:22 smithi149 ceph-mon[49797]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:23.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:22 smithi149 ceph-mon[49797]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:23.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:22 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:23.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:22 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:23.435 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:24:23.436 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:24:23.436 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:24:23.436 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:24:23.436 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:24:23.436 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:24:23.436 INFO:teuthology.orchestra.run.smithi069.stdout: "osd", 2024-04-22T23:24:23.436 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:24:23.436 INFO:teuthology.orchestra.run.smithi069.stdout: "mds", 2024-04-22T23:24:23.436 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-04-22T23:24:23.436 INFO:teuthology.orchestra.run.smithi069.stdout: "nfs" 2024-04-22T23:24:23.436 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:24:23.436 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "18/23 daemons upgraded", 2024-04-22T23:24:23.437 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading node-exporter daemons", 2024-04-22T23:24:23.437 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:24:23.437 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:24:23.855 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:24:24.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:23 smithi069 ceph-mon[63133]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:24.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:23 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/3930648281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:24.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:23 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/2345436960' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:24:24.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:23 smithi149 ceph-mon[49797]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:24.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:23 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/3930648281' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:24.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:23 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/2345436960' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:24:25.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:24 smithi069 ceph-mon[63133]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:25.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:24 smithi069 ceph-mon[63133]: pgmap v349: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:25.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:24 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:25.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:24 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:25.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:24 smithi149 ceph-mon[49797]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:25.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:24 smithi149 ceph-mon[49797]: pgmap v349: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:25.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:24 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:25.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:24 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:26.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:26 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:26.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:26 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:26.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:26 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:26.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:26 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:27.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:27 smithi149 ceph-mon[49797]: pgmap v350: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:27.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:27 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:27.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:27 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:27.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:27 smithi069 ceph-mon[63133]: pgmap v350: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:27.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:27 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:27.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:27 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:28.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:28 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:28.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:28 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:28.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:28 smithi069 ceph-mon[63133]: pgmap v351: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:28.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:28 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:28.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:28 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:28.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:28 smithi149 ceph-mon[49797]: pgmap v351: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: pgmap v352: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:24:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:24:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:24:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:30 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:24:31.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: pgmap v352: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:24:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:24:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:24:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:31.202 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:30 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:24:33.140 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:32 smithi069 ceph-mon[63133]: pgmap v353: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:24:33.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:32 smithi149 ceph-mon[49797]: pgmap v353: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:24:34.151 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:33 smithi069 ceph-mon[63133]: Upgrade: Updating prometheus.smithi069 2024-04-22T23:24:34.151 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:33 smithi069 ceph-mon[63133]: Deploying daemon prometheus.smithi069 on smithi069 2024-04-22T23:24:34.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:33 smithi149 ceph-mon[49797]: Upgrade: Updating prometheus.smithi069 2024-04-22T23:24:34.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:33 smithi149 ceph-mon[49797]: Deploying daemon prometheus.smithi069 on smithi069 2024-04-22T23:24:35.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:34 smithi069 ceph-mon[63133]: pgmap v354: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:35.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:34 smithi149 ceph-mon[49797]: pgmap v354: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:37.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:36 smithi149 ceph-mon[49797]: pgmap v355: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:37.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:36 smithi069 ceph-mon[63133]: pgmap v355: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:38 smithi149 ceph-mon[49797]: pgmap v356: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:39.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:38 smithi069 ceph-mon[63133]: pgmap v356: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:41.141 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:40 smithi069 ceph-mon[63133]: pgmap v357: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:40 smithi149 ceph-mon[49797]: pgmap v357: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:43.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:42 smithi149 ceph-mon[49797]: pgmap v358: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:24:43.379 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:42 smithi069 ceph-mon[63133]: pgmap v358: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:24:45.157 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:44 smithi069 ceph-mon[63133]: pgmap v359: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:45.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:44 smithi149 ceph-mon[49797]: pgmap v359: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:47.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:47.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:47.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:24:47.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:24:47.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:46 smithi069 ceph-mon[63133]: pgmap v360: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:47.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:47.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:47.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:47.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:47.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:24:47.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:24:47.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:46 smithi149 ceph-mon[49797]: pgmap v360: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:47.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:47.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:49.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:48 smithi069 ceph-mon[63133]: pgmap v361: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:49.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:48 smithi149 ceph-mon[49797]: pgmap v361: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:51.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:50 smithi069 ceph-mon[63133]: pgmap v362: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:51.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:50 smithi149 ceph-mon[49797]: pgmap v362: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:52.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:52 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:52.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:52 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:52.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:52 smithi069 ceph-mon[63133]: pgmap v363: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:24:52.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:52 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:52.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:52 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:52.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:52 smithi149 ceph-mon[49797]: pgmap v363: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:24:53.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:53 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:53.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:53 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:53.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:53 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:53.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:53 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:54.212 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:24:54.839 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:54 smithi069 ceph-mon[63133]: pgmap v364: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:54.839 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:54 smithi069 ceph-mon[63133]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:54.886 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:24:54.886 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (11m) 3s ago 20m 25.9M - 0.23.0 ba2b418f427c 28339322ceaa 2024-04-22T23:24:54.886 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (7m) 3s ago 20m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:24:54.886 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (7m) 32s ago 19m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:24:54.886 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (10m) 3s ago 19m 54.2M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (109s) 3s ago 14m 35.8M - 19.0.0-3252-g43be0201 a0434699c1d7 f04c8fddac95 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (80s) 32s ago 14m 15.1M - 19.0.0-3252-g43be0201 a0434699c1d7 a44cb1d5a336 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (10m) 3s ago 21m 518M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (9m) 32s ago 19m 453M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (9m) 3s ago 21m 62.9M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (8m) 32s ago 19m 47.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (66s) 3s ago 13m 68.8M - 5.7 a0434699c1d7 c41386c372b1 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (60s) 32s ago 13m 69.0M - 5.7 a0434699c1d7 9f0fcbc0fa9a 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (43s) 3s ago 19m 7746k - 1.5.0 0da6a335fe13 361a4271e5d1 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (35s) 32s ago 19m 3594k - 1.5.0 0da6a335fe13 90874bf23241 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (7m) 3s ago 18m 436M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (6m) 3s ago 18m 399M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (5m) 3s ago 17m 307M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (5m) 3s ago 17m 356M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 e7772e3632e1 2024-04-22T23:24:54.887 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (4m) 32s ago 16m 376M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 036a1a8ba3c9 2024-04-22T23:24:54.888 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (3m) 32s ago 16m 339M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 539c90bd7194 2024-04-22T23:24:54.888 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (2m) 32s ago 16m 347M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 b821ba1ee812 2024-04-22T23:24:54.888 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (2m) 32s ago 15m 334M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 fec23dae1ff2 2024-04-22T23:24:54.888 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (9s) 3s ago 19m 45.4M - 2.43.0 a07b618ecd1d 970d1d833a75 2024-04-22T23:24:54.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:54 smithi149 ceph-mon[49797]: pgmap v364: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:54.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:54 smithi149 ceph-mon[49797]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:55.303 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:24:55.303 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:24:55.303 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:24:55.303 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 8 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 14 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:24:55.304 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:24:55.727 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:24:55.727 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:24:55.727 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:24:55.727 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:24:55.727 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:24:55.727 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:24:55.727 INFO:teuthology.orchestra.run.smithi069.stdout: "osd", 2024-04-22T23:24:55.727 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:24:55.727 INFO:teuthology.orchestra.run.smithi069.stdout: "mds", 2024-04-22T23:24:55.727 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-04-22T23:24:55.728 INFO:teuthology.orchestra.run.smithi069.stdout: "nfs" 2024-04-22T23:24:55.728 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:24:55.728 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "18/23 daemons upgraded", 2024-04-22T23:24:55.728 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading prometheus daemons", 2024-04-22T23:24:55.728 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:24:55.728 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:24:55.852 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:55 smithi149 ceph-mon[49797]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:55.852 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:55 smithi149 ceph-mon[49797]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:55.852 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:55 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/2820266807' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:55.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:55 smithi069 ceph-mon[63133]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:55.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:55 smithi069 ceph-mon[63133]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:55.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:55 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/2820266807' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.133 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:24:56.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:56.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: pgmap v365: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:56.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/3918159308' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:24:56.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:56.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:56.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:24:56.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:24:56.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:56.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:56.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:24:56.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:56 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: pgmap v365: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/3918159308' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:56.952 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:56 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:24:57.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:57 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:24:57.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:57 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T23:24:58.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:58 smithi069 ceph-mon[63133]: pgmap v366: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:58.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:58 smithi069 ceph-mon[63133]: Upgrade: Updating alertmanager.smithi069 2024-04-22T23:24:58.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:24:58 smithi069 ceph-mon[63133]: Deploying daemon alertmanager.smithi069 on smithi069 2024-04-22T23:24:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:58 smithi149 ceph-mon[49797]: pgmap v366: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:24:58.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:58 smithi149 ceph-mon[49797]: Upgrade: Updating alertmanager.smithi069 2024-04-22T23:24:58.951 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:24:58 smithi149 ceph-mon[49797]: Deploying daemon alertmanager.smithi069 on smithi069 2024-04-22T23:25:01.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:00 smithi069 ceph-mon[63133]: pgmap v367: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:01.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:00 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:25:01.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:00 smithi149 ceph-mon[49797]: pgmap v367: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:01.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:00 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:25:03.138 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:02 smithi069 ceph-mon[63133]: pgmap v368: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:25:03.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:02 smithi149 ceph-mon[49797]: pgmap v368: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:25:05.157 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:04 smithi069 ceph-mon[63133]: pgmap v369: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:05.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:04 smithi149 ceph-mon[49797]: pgmap v369: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:07.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:06 smithi149 ceph-mon[49797]: pgmap v370: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:07.358 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:06 smithi069 ceph-mon[63133]: pgmap v370: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:08.971 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:08 smithi069 ceph-mon[63133]: pgmap v371: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:09.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:08 smithi149 ceph-mon[49797]: pgmap v371: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:10.292 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:10 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:10.293 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:10 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:10.293 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:10 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:25:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:10 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:10.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:10 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:10.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:10 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:25:11.557 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:11 smithi069 ceph-mon[63133]: pgmap v372: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:11.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:11 smithi149 ceph-mon[49797]: pgmap v372: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:13.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:13 smithi069 ceph-mon[63133]: pgmap v373: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:25:13.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:13 smithi149 ceph-mon[49797]: pgmap v373: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:25:15.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:15 smithi069 ceph-mon[63133]: pgmap v374: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:15.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:15 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:15.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:15 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:15.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:15 smithi149 ceph-mon[49797]: pgmap v374: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:15.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:15 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:15.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:15 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:16.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:16.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:16.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:25:16.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:16.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:16.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:25:17.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:17 smithi069 ceph-mon[63133]: pgmap v375: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:17.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:17 smithi149 ceph-mon[49797]: pgmap v375: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:19.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: pgmap v376: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:19.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:19.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:19.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:25:19.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:25:19.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:19.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:19.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T23:25:19.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi069:9093"}]: dispatch 2024-04-22T23:25:19.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:19.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:25:19.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:19 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: pgmap v376: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:19.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:19.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:19.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:25:19.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi069:9093"}]: dispatch 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:19.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:19 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:20.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:20 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T23:25:20.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:20 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi069:9093"}]: dispatch 2024-04-22T23:25:20.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:20 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T23:25:20.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:20 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi069:9093"}]: dispatch 2024-04-22T23:25:21.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:21 smithi069 ceph-mon[63133]: pgmap v377: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:21.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:21 smithi149 ceph-mon[49797]: pgmap v377: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:22.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:22 smithi069 ceph-mon[63133]: Upgrade: Updating grafana.smithi069 2024-04-22T23:25:22.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:22 smithi069 ceph-mon[63133]: Deploying daemon grafana.smithi069 on smithi069 2024-04-22T23:25:22.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:22 smithi149 ceph-mon[49797]: Upgrade: Updating grafana.smithi069 2024-04-22T23:25:22.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:22 smithi149 ceph-mon[49797]: Deploying daemon grafana.smithi069 on smithi069 2024-04-22T23:25:23.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:23 smithi069 ceph-mon[63133]: pgmap v378: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:25:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:23 smithi149 ceph-mon[49797]: pgmap v378: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:25:25.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:25 smithi069 ceph-mon[63133]: pgmap v379: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:25 smithi149 ceph-mon[49797]: pgmap v379: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:26.526 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:25:27.284 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:25:27.284 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (18s) 12s ago 20m 14.7M - 0.25.0 c8568f914cd2 0c7526d9f598 2024-04-22T23:25:27.284 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (8m) 12s ago 20m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:25:27.284 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (8m) 64s ago 20m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:25:27.284 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (11m) 12s ago 20m 54.9M - 8.3.5 dad864ee21e9 4462dc456e1c 2024-04-22T23:25:27.284 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (2m) 12s ago 14m 36.0M - 19.0.0-3252-g43be0201 a0434699c1d7 f04c8fddac95 2024-04-22T23:25:27.284 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (112s) 64s ago 14m 15.1M - 19.0.0-3252-g43be0201 a0434699c1d7 a44cb1d5a336 2024-04-22T23:25:27.285 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (11m) 12s ago 22m 518M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:25:27.285 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (10m) 64s ago 20m 453M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:25:27.285 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (9m) 12s ago 22m 63.7M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:25:27.285 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (9m) 64s ago 20m 47.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:25:27.285 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (99s) 12s ago 14m 68.9M - 5.7 a0434699c1d7 c41386c372b1 2024-04-22T23:25:27.285 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (93s) 64s ago 14m 69.0M - 5.7 a0434699c1d7 9f0fcbc0fa9a 2024-04-22T23:25:27.285 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (75s) 12s ago 20m 7642k - 1.5.0 0da6a335fe13 361a4271e5d1 2024-04-22T23:25:27.285 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (67s) 64s ago 19m 3594k - 1.5.0 0da6a335fe13 90874bf23241 2024-04-22T23:25:27.285 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (7m) 12s ago 19m 436M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:25:27.285 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (6m) 12s ago 18m 399M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:25:27.285 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (6m) 12s ago 18m 308M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:25:27.285 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (5m) 12s ago 17m 356M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 e7772e3632e1 2024-04-22T23:25:27.286 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (4m) 64s ago 17m 376M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 036a1a8ba3c9 2024-04-22T23:25:27.286 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (4m) 64s ago 17m 339M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 539c90bd7194 2024-04-22T23:25:27.286 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (3m) 64s ago 16m 347M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 b821ba1ee812 2024-04-22T23:25:27.286 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (2m) 64s ago 16m 334M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 fec23dae1ff2 2024-04-22T23:25:27.286 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (41s) 12s ago 19m 51.2M - 2.43.0 a07b618ecd1d 970d1d833a75 2024-04-22T23:25:27.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:27 smithi069 ceph-mon[63133]: pgmap v380: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:27.647 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:27 smithi149 ceph-mon[49797]: pgmap v380: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:27.686 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:25:27.686 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:25:27.686 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 8 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 14 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:25:27.687 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:25:28.031 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:25:28.031 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", 2024-04-22T23:25:28.031 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-04-22T23:25:28.031 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T23:25:28.031 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-04-22T23:25:28.031 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-04-22T23:25:28.032 INFO:teuthology.orchestra.run.smithi069.stdout: "osd", 2024-04-22T23:25:28.032 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-04-22T23:25:28.032 INFO:teuthology.orchestra.run.smithi069.stdout: "mds", 2024-04-22T23:25:28.032 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-04-22T23:25:28.032 INFO:teuthology.orchestra.run.smithi069.stdout: "nfs" 2024-04-22T23:25:28.032 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:25:28.032 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "18/23 daemons upgraded", 2024-04-22T23:25:28.032 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading grafana daemons", 2024-04-22T23:25:28.032 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:25:28.032 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:25:28.431 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:25:28.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:28 smithi149 ceph-mon[49797]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:25:28.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:28 smithi149 ceph-mon[49797]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:25:28.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:28 smithi149 ceph-mon[49797]: from='client.34616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:25:28.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:28 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/3206906575' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:28.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:28 smithi069 ceph-mon[63133]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:25:28.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:28 smithi069 ceph-mon[63133]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:25:28.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:28 smithi069 ceph-mon[63133]: from='client.34616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:25:28.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:28 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/3206906575' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:29.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:29 smithi149 ceph-mon[49797]: pgmap v381: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:29.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:29 smithi149 ceph-mon[49797]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:25:29.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:29 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/4059767661' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:25:29.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:29 smithi069 ceph-mon[63133]: pgmap v381: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:29.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:29 smithi069 ceph-mon[63133]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:25:29.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:29 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/4059767661' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:25:31.684 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:31 smithi069 ceph-mon[63133]: pgmap v382: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:31.684 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:25:31.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:31 smithi149 ceph-mon[49797]: pgmap v382: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:31.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:25:33.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:33 smithi069 ceph-mon[63133]: pgmap v383: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:25:33.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:33 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:33.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:33 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:33.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:33 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:25:33.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:33 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:33.396 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:33 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:33 smithi149 ceph-mon[49797]: pgmap v383: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:25:33.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:33 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:33 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:33 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:25:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:33 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:33.451 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:33 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:35.395 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:35 smithi069 ceph-mon[63133]: pgmap v384: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:35.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:35 smithi149 ceph-mon[49797]: pgmap v384: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:37.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:37 smithi069 ceph-mon[63133]: pgmap v385: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:37.450 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:37 smithi149 ceph-mon[49797]: pgmap v385: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:39.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:39.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:38 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:39.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:38 smithi069 ceph-mon[63133]: pgmap v386: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:39.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:38 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:39.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:38 smithi149 ceph-mon[49797]: pgmap v386: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:40.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:39 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:40.145 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:39 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:40.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:39 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:40.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:39 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:40.896 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:40 smithi069 ceph-mon[63133]: pgmap v387: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:41.200 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:40 smithi149 ceph-mon[49797]: pgmap v387: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:43.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: pgmap v388: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:25:43.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:25:43.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:25:43.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi069:3000"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.647 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T23:25:43.648 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T23:25:43.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T23:25:43.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T23:25:43.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T23:25:43.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T23:25:43.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T23:25:43.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-22T23:25:43.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T23:25:43.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T23:25:43.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T23:25:43.649 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T23:25:43.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T23:25:43.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-22T23:25:43.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T23:25:43.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T23:25:43.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T23:25:43.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T23:25:43.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.650 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.651 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: pgmap v388: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:25:43.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:25:43.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:25:43.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi069:3000"}]: dispatch 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.702 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.703 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.704 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.704 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.704 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:25:43.704 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.704 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T23:25:43.704 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T23:25:43.704 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.704 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T23:25:43.704 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T23:25:43.704 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T23:25:43.704 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T23:25:43.704 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T23:25:43.705 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.706 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:25:43.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:25:43.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:25:43.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:43.707 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:44.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:44 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T23:25:44.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:44 smithi069 ceph-mon[63133]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi069:3000"}]: dispatch 2024-04-22T23:25:44.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:44 smithi069 ceph-mon[63133]: Upgrade: Finalizing container_image settings 2024-04-22T23:25:44.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:44 smithi069 ceph-mon[63133]: Upgrade: Complete! 2024-04-22T23:25:44.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:44 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T23:25:44.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:44 smithi149 ceph-mon[49797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi069:3000"}]: dispatch 2024-04-22T23:25:44.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:44 smithi149 ceph-mon[49797]: Upgrade: Finalizing container_image settings 2024-04-22T23:25:44.701 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:44 smithi149 ceph-mon[49797]: Upgrade: Complete! 2024-04-22T23:25:45.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:45 smithi069 ceph-mon[63133]: pgmap v389: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:45.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:45 smithi149 ceph-mon[49797]: pgmap v389: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:46.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:25:46.646 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:46.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:25:46.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:25:47.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:47 smithi069 ceph-mon[63133]: pgmap v390: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:47.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:47 smithi149 ceph-mon[49797]: pgmap v390: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:49.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:49 smithi069 ceph-mon[63133]: pgmap v391: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:49.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:49 smithi149 ceph-mon[49797]: pgmap v391: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:51.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:51 smithi069 ceph-mon[63133]: pgmap v392: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:51.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:51 smithi149 ceph-mon[49797]: pgmap v392: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:53.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:53 smithi069 ceph-mon[63133]: pgmap v393: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:25:53.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:53 smithi149 ceph-mon[49797]: pgmap v393: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:25:55.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:55 smithi069 ceph-mon[63133]: pgmap v394: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:55.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:55 smithi149 ceph-mon[49797]: pgmap v394: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:57.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:57 smithi069 ceph-mon[63133]: pgmap v395: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:57.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:57 smithi149 ceph-mon[49797]: pgmap v395: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:59.422 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph orch ps' 2024-04-22T23:25:59.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:25:59 smithi069 ceph-mon[63133]: pgmap v396: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:25:59.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:25:59 smithi149 ceph-mon[49797]: pgmap v396: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:00.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:00 smithi069 ceph-mon[63133]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:26:00.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:00 smithi149 ceph-mon[49797]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:26:01.407 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:26:01.407 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (52s) 23s ago 21m 14.8M - 0.25.0 c8568f914cd2 0c7526d9f598 2024-04-22T23:26:01.407 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (8m) 23s ago 21m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:26:01.407 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (8m) 98s ago 20m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:26:01.407 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (29s) 23s ago 21m 50.8M - 9.4.12 2bacad6d85d8 02a4987fd9c8 2024-04-22T23:26:01.407 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (2m) 23s ago 15m 36.2M - 19.0.0-3252-g43be0201 a0434699c1d7 f04c8fddac95 2024-04-22T23:26:01.407 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (2m) 98s ago 15m 15.1M - 19.0.0-3252-g43be0201 a0434699c1d7 a44cb1d5a336 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (11m) 23s ago 22m 525M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (10m) 98s ago 20m 453M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (10m) 23s ago 22m 64.2M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (9m) 98s ago 20m 47.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (2m) 23s ago 15m 69.0M - 5.7 a0434699c1d7 c41386c372b1 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (2m) 98s ago 14m 69.0M - 5.7 a0434699c1d7 9f0fcbc0fa9a 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (109s) 23s ago 21m 7810k - 1.5.0 0da6a335fe13 361a4271e5d1 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (102s) 98s ago 20m 3594k - 1.5.0 0da6a335fe13 90874bf23241 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (8m) 23s ago 19m 437M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (7m) 23s ago 19m 399M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (6m) 23s ago 18m 308M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (6m) 23s ago 18m 357M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 e7772e3632e1 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (5m) 98s ago 18m 376M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 036a1a8ba3c9 2024-04-22T23:26:01.408 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (4m) 98s ago 17m 339M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 539c90bd7194 2024-04-22T23:26:01.409 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (4m) 98s ago 17m 347M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 b821ba1ee812 2024-04-22T23:26:01.409 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (3m) 98s ago 16m 334M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 fec23dae1ff2 2024-04-22T23:26:01.409 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (76s) 23s ago 20m 53.7M - 2.43.0 a07b618ecd1d 970d1d833a75 2024-04-22T23:26:01.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:01 smithi069 ceph-mon[63133]: pgmap v397: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:01.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:01 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:26:01.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:01 smithi149 ceph-mon[49797]: pgmap v397: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:01.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:01 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:26:01.995 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph versions' 2024-04-22T23:26:02.574 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:02 smithi069 ceph-mon[63133]: from='client.34636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:26:02.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:02 smithi149 ceph-mon[49797]: from='client.34636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:26:03.579 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:03 smithi069 ceph-mon[63133]: pgmap v398: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:26:03.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:03 smithi149 ceph-mon[49797]: pgmap v398: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:26:04.092 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:26:04.092 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:26:04.092 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:26:04.093 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:26:04.093 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:26:04.093 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:26:04.093 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:26:04.093 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:26:04.093 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 8 2024-04-22T23:26:04.093 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:26:04.093 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:26:04.093 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:26:04.093 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:26:04.093 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:26:04.093 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 14 2024-04-22T23:26:04.094 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:26:04.094 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:26:04.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:04 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/2712460405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:26:04.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:04 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/2712460405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:26:04.777 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-22T23:26:05.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:05 smithi069 ceph-mon[63133]: pgmap v399: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:05.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:05 smithi149 ceph-mon[49797]: pgmap v399: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:06.461 INFO:teuthology.orchestra.run.smithi069.stdout:wait for servicemap items w/ changing names to refresh 2024-04-22T23:26:07.027 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'sleep 60' 2024-04-22T23:26:07.612 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:07 smithi069 ceph-mon[63133]: pgmap v400: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:07.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:07 smithi149 ceph-mon[49797]: pgmap v400: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:09.645 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:09 smithi069 ceph-mon[63133]: pgmap v401: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:09.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:09 smithi149 ceph-mon[49797]: pgmap v401: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:11.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:11 smithi149 ceph-mon[49797]: pgmap v402: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:11.897 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:11 smithi069 ceph-mon[63133]: pgmap v402: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:13.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:13 smithi149 ceph-mon[49797]: pgmap v403: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:26:13.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:13 smithi069 ceph-mon[63133]: pgmap v403: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:26:15.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:15 smithi149 ceph-mon[49797]: pgmap v404: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:15.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:15 smithi069 ceph-mon[63133]: pgmap v404: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:16.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:16 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:26:16.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:16 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:26:17.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:17 smithi149 ceph-mon[49797]: pgmap v405: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:17.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:17 smithi069 ceph-mon[63133]: pgmap v405: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:19.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:19 smithi149 ceph-mon[49797]: pgmap v406: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:19.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:19 smithi069 ceph-mon[63133]: pgmap v406: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:21.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:21 smithi149 ceph-mon[49797]: pgmap v407: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:21.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:21 smithi069 ceph-mon[63133]: pgmap v407: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:23.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:23 smithi149 ceph-mon[49797]: pgmap v408: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:26:23.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:23 smithi069 ceph-mon[63133]: pgmap v408: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:26:25.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:25 smithi149 ceph-mon[49797]: pgmap v409: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:25.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:25 smithi069 ceph-mon[63133]: pgmap v409: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:27.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:27 smithi149 ceph-mon[49797]: pgmap v410: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:27.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:27 smithi069 ceph-mon[63133]: pgmap v410: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:29.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:29 smithi149 ceph-mon[49797]: pgmap v411: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:29.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:29 smithi069 ceph-mon[63133]: pgmap v411: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:31.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:31 smithi149 ceph-mon[49797]: pgmap v412: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:31.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:26:31.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:31 smithi069 ceph-mon[63133]: pgmap v412: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:31.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:26:33.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:33 smithi149 ceph-mon[49797]: pgmap v413: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:26:33.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:33 smithi069 ceph-mon[63133]: pgmap v413: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:26:35.699 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:35 smithi149 ceph-mon[49797]: pgmap v414: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:35.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:35 smithi069 ceph-mon[63133]: pgmap v414: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:37.699 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:37 smithi149 ceph-mon[49797]: pgmap v415: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:37.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:37 smithi069 ceph-mon[63133]: pgmap v415: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:39.700 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:39 smithi149 ceph-mon[49797]: pgmap v416: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:39.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:39 smithi069 ceph-mon[63133]: pgmap v416: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:41.699 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:41 smithi149 ceph-mon[49797]: pgmap v417: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:41.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:41 smithi069 ceph-mon[63133]: pgmap v417: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:43.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:43 smithi069 ceph-mon[63133]: pgmap v418: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:26:43.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:26:43.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:26:43.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:26:43.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:26:43.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:26:43.895 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:43 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:26:43.949 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:43 smithi149 ceph-mon[49797]: pgmap v418: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:26:43.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T23:26:43.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2024-04-22T23:26:43.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T23:26:43.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T23:26:43.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:26:43.950 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:43 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' 2024-04-22T23:26:45.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:45 smithi069 ceph-mon[63133]: pgmap v419: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:45.949 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:45 smithi149 ceph-mon[49797]: pgmap v419: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:46.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:46 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:26:46.949 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:46 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:26:47.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:47 smithi069 ceph-mon[63133]: pgmap v420: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:47.949 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:47 smithi149 ceph-mon[49797]: pgmap v420: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:48.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:48 smithi069 ceph-mon[63133]: pgmap v421: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:48.949 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:48 smithi149 ceph-mon[49797]: pgmap v421: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:51.199 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:50 smithi149 ceph-mon[49797]: pgmap v422: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:51.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:50 smithi069 ceph-mon[63133]: pgmap v422: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:53.199 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:52 smithi149 ceph-mon[49797]: pgmap v423: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:26:53.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:52 smithi069 ceph-mon[63133]: pgmap v423: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:26:55.201 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:54 smithi149 ceph-mon[49797]: pgmap v424: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:55.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:54 smithi069 ceph-mon[63133]: pgmap v424: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:57.199 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:56 smithi149 ceph-mon[49797]: pgmap v425: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:57.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:56 smithi069 ceph-mon[63133]: pgmap v425: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:59.199 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:26:58 smithi149 ceph-mon[49797]: pgmap v426: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:26:59.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:26:58 smithi069 ceph-mon[63133]: pgmap v426: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:01.199 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:00 smithi149 ceph-mon[49797]: pgmap v427: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:01.199 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:00 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:27:01.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:00 smithi069 ceph-mon[63133]: pgmap v427: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:01.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:00 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:27:03.199 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:02 smithi149 ceph-mon[49797]: pgmap v428: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:27:03.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:02 smithi069 ceph-mon[63133]: pgmap v428: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:27:05.199 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:04 smithi149 ceph-mon[49797]: pgmap v429: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:05.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:04 smithi069 ceph-mon[63133]: pgmap v429: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:07.199 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:06 smithi149 ceph-mon[49797]: pgmap v430: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:07.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:06 smithi069 ceph-mon[63133]: pgmap v430: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:09.217 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:08 smithi069 ceph-mon[63133]: pgmap v431: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:09.245 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph orch ps' 2024-04-22T23:27:09.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:08 smithi149 ceph-mon[49797]: pgmap v431: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:11.119 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T23:27:11.119 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (2m) 93s ago 22m 14.8M - 0.25.0 c8568f914cd2 0c7526d9f598 2024-04-22T23:27:11.119 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (10m) 93s ago 22m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 dfca1eb58aba 2024-04-22T23:27:11.119 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi149 smithi149 running (10m) 2m ago 21m 6656k - 19.0.0-3252-g43be0201 a0434699c1d7 643db547604f 2024-04-22T23:27:11.119 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (99s) 93s ago 22m 50.8M - 9.4.12 2bacad6d85d8 02a4987fd9c8 2024-04-22T23:27:11.119 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.jiyres smithi069 running (4m) 93s ago 16m 36.2M - 19.0.0-3252-g43be0201 a0434699c1d7 f04c8fddac95 2024-04-22T23:27:11.119 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi149.ickhha smithi149 running (3m) 2m ago 16m 15.1M - 19.0.0-3252-g43be0201 a0434699c1d7 a44cb1d5a336 2024-04-22T23:27:11.120 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.lyiiqg smithi069 *:8443,9283,8765 running (12m) 93s ago 23m 525M - 19.0.0-3252-g43be0201 a0434699c1d7 9fd22cdf46ce 2024-04-22T23:27:11.120 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi149.rxqkqk smithi149 *:8443,9283,8765 running (11m) 2m ago 21m 453M - 19.0.0-3252-g43be0201 a0434699c1d7 988ba2b1225b 2024-04-22T23:27:11.120 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (11m) 93s ago 23m 64.2M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 e4e20d532cda 2024-04-22T23:27:11.120 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi149 smithi149 running (11m) 2m ago 21m 47.1M 2048M 19.0.0-3252-g43be0201 a0434699c1d7 c208daf7eb1c 2024-04-22T23:27:11.120 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.cdmddj smithi069 *:2049 running (3m) 93s ago 16m 69.0M - 5.7 a0434699c1d7 c41386c372b1 2024-04-22T23:27:11.120 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi149.mxpcoc smithi149 *:2049 running (3m) 2m ago 16m 69.0M - 5.7 a0434699c1d7 9f0fcbc0fa9a 2024-04-22T23:27:11.120 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (2m) 93s ago 22m 7810k - 1.5.0 0da6a335fe13 361a4271e5d1 2024-04-22T23:27:11.120 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi149 smithi149 *:9100 running (2m) 2m ago 21m 3594k - 1.5.0 0da6a335fe13 90874bf23241 2024-04-22T23:27:11.120 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (9m) 93s ago 21m 437M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 881ec8fb453d 2024-04-22T23:27:11.120 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (8m) 93s ago 20m 399M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 93d880f30ae4 2024-04-22T23:27:11.120 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (7m) 93s ago 20m 308M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 8d3d66dfd7f9 2024-04-22T23:27:11.120 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (7m) 93s ago 19m 357M 1936M 19.0.0-3252-g43be0201 a0434699c1d7 e7772e3632e1 2024-04-22T23:27:11.121 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi149 running (6m) 2m ago 19m 376M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 036a1a8ba3c9 2024-04-22T23:27:11.121 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi149 running (5m) 2m ago 18m 339M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 539c90bd7194 2024-04-22T23:27:11.121 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi149 running (5m) 2m ago 18m 347M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 b821ba1ee812 2024-04-22T23:27:11.121 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi149 running (4m) 2m ago 18m 334M 4096M 19.0.0-3252-g43be0201 a0434699c1d7 fec23dae1ff2 2024-04-22T23:27:11.121 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (2m) 93s ago 21m 53.7M - 2.43.0 a07b618ecd1d 970d1d833a75 2024-04-22T23:27:11.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:10 smithi069 ceph-mon[63133]: pgmap v432: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:11.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:10 smithi149 ceph-mon[49797]: pgmap v432: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:11.789 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph orch upgrade status' 2024-04-22T23:27:12.091 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:11 smithi069 ceph-mon[63133]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:27:12.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:11 smithi149 ceph-mon[49797]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:27:13.277 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:12 smithi069 ceph-mon[63133]: pgmap v433: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:27:13.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:12 smithi149 ceph-mon[49797]: pgmap v433: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:27:13.761 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:27:13.762 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": null, 2024-04-22T23:27:13.762 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": false, 2024-04-22T23:27:13.762 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "", 2024-04-22T23:27:13.762 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [], 2024-04-22T23:27:13.762 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": null, 2024-04-22T23:27:13.762 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "", 2024-04-22T23:27:13.762 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-04-22T23:27:13.762 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:27:14.358 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph health detail' 2024-04-22T23:27:15.317 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:14 smithi069 ceph-mon[63133]: from='client.34648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:27:15.318 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:14 smithi069 ceph-mon[63133]: pgmap v434: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:15.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:14 smithi149 ceph-mon[49797]: from='client.34648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:27:15.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:14 smithi149 ceph-mon[49797]: pgmap v434: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:16.144 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:15 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:27:16.384 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-04-22T23:27:16.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:15 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:27:17.016 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph versions' 2024-04-22T23:27:17.319 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:16 smithi069 ceph-mon[63133]: pgmap v435: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:17.319 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:16 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/4151219120' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:27:17.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:16 smithi149 ceph-mon[49797]: pgmap v435: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:17.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:16 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/4151219120' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T23:27:19.171 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-22T23:27:19.171 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-04-22T23:27:19.171 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:27:19.171 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:27:19.171 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-04-22T23:27:19.172 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:27:19.172 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:27:19.172 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-04-22T23:27:19.172 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 8 2024-04-22T23:27:19.172 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:27:19.172 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-04-22T23:27:19.172 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 2 2024-04-22T23:27:19.172 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:27:19.172 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-04-22T23:27:19.172 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)": 14 2024-04-22T23:27:19.172 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:27:19.173 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-22T23:27:19.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:18 smithi069 ceph-mon[63133]: pgmap v436: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:19.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:18 smithi149 ceph-mon[49797]: pgmap v436: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:19.787 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-22T23:27:20.088 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:19 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/574046950' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:27:20.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:19 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/574046950' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:27:21.356 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:21 smithi069 ceph-mon[63133]: pgmap v437: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:21.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:21 smithi149 ceph-mon[49797]: pgmap v437: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:21.867 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-04-22T23:27:22.144 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:22 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/575599256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:27:22.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:22 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/575599256' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:27:22.459 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -e sha1=43be020184947e53516056c9931e1ac5bdbbb1a5 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-22T23:27:23.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:23 smithi069 ceph-mon[63133]: pgmap v438: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:27:23.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:23 smithi149 ceph-mon[49797]: pgmap v438: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:27:24.513 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-3252-g43be0201 (43be020184947e53516056c9931e1ac5bdbbb1a5) squid (dev)" 2024-04-22T23:27:25.091 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-04-22T23:27:25.091 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-04-22T23:27:25.092 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-22T23:27:25.392 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:25 smithi069 ceph-mon[63133]: pgmap v439: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:25.392 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:25 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/1934000787' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:27:25.448 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:25 smithi149 ceph-mon[49797]: pgmap v439: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:25.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:25 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/1934000787' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T23:27:27.066 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-22T23:27:27.067 INFO:teuthology.orchestra.run.smithi069.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-22T23:04:02.398445Z", "last_refresh": "2024-04-22T23:25:37.707557Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-22T23:03:57.283881Z", "last_refresh": "2024-04-22T23:24:22.719665Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-04-22T23:03:59.884436Z", "last_refresh": "2024-04-22T23:25:37.707702Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-22T23:10:32.123823Z", "last_refresh": "2024-04-22T23:24:22.720866Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-22T23:03:55.163282Z", "last_refresh": "2024-04-22T23:24:22.719827Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi069:172.21.15.69=smithi069", "smithi149:172.21.15.149=smithi149"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-22T23:04:47.223932Z", "last_refresh": "2024-04-22T23:24:22.719966Z", "running": 2, "size": 2}}, {"events": ["2024-04-22T23:26:43.420871Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-04-22T23:10:53.248224Z", "last_refresh": "2024-04-22T23:24:22.721005Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-22T23:04:01.114168Z", "last_refresh": "2024-04-22T23:24:22.720118Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c", "last_refresh": "2024-04-22T23:24:22.720291Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-22T23:03:58.572865Z", "last_refresh": "2024-04-22T23:25:37.707845Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-22T23:27:27.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:27 smithi069 ceph-mon[63133]: pgmap v440: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:27.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:27 smithi149 ceph-mon[49797]: pgmap v440: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:27.753 INFO:tasks.cephadm:nfs.foo has 2/2 2024-04-22T23:27:27.754 DEBUG:teuthology.parallel:result is None 2024-04-22T23:27:27.754 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-22T23:27:27.763 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T23:27:27.763 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-04-22T23:27:27.787 INFO:teuthology.orchestra.run.smithi069.stderr:+ umount /mnt/foo 2024-04-22T23:27:27.799 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T23:27:27.808 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T23:27:27.808 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-04-22T23:27:28.374 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:28 smithi069 ceph-mon[63133]: from='client.34668 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:27:28.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:28 smithi149 ceph-mon[49797]: from='client.34668 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T23:27:29.389 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:29 smithi069 ceph-mon[63133]: pgmap v441: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:29.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:29 smithi149 ceph-mon[49797]: pgmap v441: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:29.883 INFO:teuthology.orchestra.run.smithi069.stdout: "foo" 2024-04-22T23:27:30.492 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-04-22T23:27:31.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:31 smithi069 ceph-mon[63133]: from='client.34672 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:27:31.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:31 smithi069 ceph-mon[63133]: pgmap v442: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:31.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:31 smithi069 ceph-mon[63133]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:27:31.448 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:31 smithi149 ceph-mon[49797]: from='client.34672 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:27:31.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:31 smithi149 ceph-mon[49797]: pgmap v442: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T23:27:31.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:31 smithi149 ceph-mon[49797]: from='mgr.34104 172.21.15.69:0/3491809138' entity='mgr.smithi069.lyiiqg' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T23:27:32.667 INFO:teuthology.orchestra.run.smithi069.stdout:[ 2024-04-22T23:27:32.667 INFO:teuthology.orchestra.run.smithi069.stdout: { 2024-04-22T23:27:32.667 INFO:teuthology.orchestra.run.smithi069.stdout: "access_type": "RW", 2024-04-22T23:27:32.667 INFO:teuthology.orchestra.run.smithi069.stdout: "clients": [], 2024-04-22T23:27:32.667 INFO:teuthology.orchestra.run.smithi069.stdout: "cluster_id": "foo", 2024-04-22T23:27:32.667 INFO:teuthology.orchestra.run.smithi069.stdout: "export_id": 1, 2024-04-22T23:27:32.667 INFO:teuthology.orchestra.run.smithi069.stdout: "fsal": { 2024-04-22T23:27:32.667 INFO:teuthology.orchestra.run.smithi069.stdout: "fs_name": "foofs", 2024-04-22T23:27:32.667 INFO:teuthology.orchestra.run.smithi069.stdout: "name": "CEPH", 2024-04-22T23:27:32.667 INFO:teuthology.orchestra.run.smithi069.stdout: "user_id": "nfs.foo.1" 2024-04-22T23:27:32.667 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-04-22T23:27:32.668 INFO:teuthology.orchestra.run.smithi069.stdout: "path": "/", 2024-04-22T23:27:32.668 INFO:teuthology.orchestra.run.smithi069.stdout: "protocols": [ 2024-04-22T23:27:32.668 INFO:teuthology.orchestra.run.smithi069.stdout: 4 2024-04-22T23:27:32.668 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-04-22T23:27:32.668 INFO:teuthology.orchestra.run.smithi069.stdout: "pseudo": "/fake", 2024-04-22T23:27:32.668 INFO:teuthology.orchestra.run.smithi069.stdout: "security_label": true, 2024-04-22T23:27:32.668 INFO:teuthology.orchestra.run.smithi069.stdout: "squash": "none", 2024-04-22T23:27:32.668 INFO:teuthology.orchestra.run.smithi069.stdout: "transports": [ 2024-04-22T23:27:32.668 INFO:teuthology.orchestra.run.smithi069.stdout: "TCP" 2024-04-22T23:27:32.668 INFO:teuthology.orchestra.run.smithi069.stdout: ] 2024-04-22T23:27:32.668 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-04-22T23:27:32.668 INFO:teuthology.orchestra.run.smithi069.stdout:] 2024-04-22T23:27:33.308 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:33 smithi069 ceph-mon[63133]: pgmap v443: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:27:33.331 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-04-22T23:27:33.448 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:33 smithi149 ceph-mon[49797]: pgmap v443: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T23:27:34.224 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:34 smithi069 ceph-mon[63133]: from='client.34676 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:27:34.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:34 smithi149 ceph-mon[49797]: from='client.34676 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T23:27:35.018 INFO:teuthology.orchestra.run.smithi069.stdout:foo rec-0000000000000004:nfs.foo.1 2024-04-22T23:27:35.018 INFO:teuthology.orchestra.run.smithi069.stdout:foo grace 2024-04-22T23:27:35.018 INFO:teuthology.orchestra.run.smithi069.stdout:foo rec-0000000000000004:nfs.foo.0 2024-04-22T23:27:35.018 INFO:teuthology.orchestra.run.smithi069.stdout:foo conf-nfs.foo 2024-04-22T23:27:35.018 INFO:teuthology.orchestra.run.smithi069.stdout:foo export-1 2024-04-22T23:27:35.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:35 smithi069 ceph-mon[63133]: pgmap v444: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T23:27:35.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:35 smithi069 ceph-mon[63133]: mgrmap e42: smithi069.lyiiqg(active, since 11m), standbys: smithi149.rxqkqk 2024-04-22T23:27:35.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:35 smithi149 ceph-mon[49797]: pgmap v444: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T23:27:35.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:35 smithi149 ceph-mon[49797]: mgrmap e42: smithi069.lyiiqg(active, since 11m), standbys: smithi149.rxqkqk 2024-04-22T23:27:35.704 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T23:27:35.715 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T23:27:35.715 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 -- bash -c 'set -ex 2024-04-22T23:27:35.715 DEBUG:teuthology.orchestra.run.smithi069:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-04-22T23:27:35.715 DEBUG:teuthology.orchestra.run.smithi069:> ' 2024-04-22T23:27:37.218 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:37 smithi069 ceph-mon[63133]: pgmap v445: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T23:27:37.423 INFO:teuthology.orchestra.run.smithi069.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-04-22T23:27:37.448 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:37 smithi149 ceph-mon[49797]: pgmap v445: 97 pgs: 97 active+clean; 238 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T23:27:37.657 INFO:teuthology.orchestra.run.smithi069.stderr:+ [[ 6 -gt 2 ]] 2024-04-22T23:27:38.251 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-22T23:27:38.261 INFO:tasks.cephadm:Teardown begin 2024-04-22T23:27:38.262 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T23:27:38.283 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:38 smithi069 ceph-mon[63133]: from='client.? 172.21.15.69:0/1515937176' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-22T23:27:38.287 DEBUG:teuthology.orchestra.run.smithi149:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T23:27:38.354 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-22T23:27:38.354 DEBUG:teuthology.orchestra.run.smithi069:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-22T23:27:38.369 DEBUG:teuthology.orchestra.run.smithi149:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-22T23:27:38.387 INFO:tasks.cephadm:Stopping all daemons... 2024-04-22T23:27:38.387 INFO:tasks.cephadm.mon.smithi069:Stopping mon.smithi069... 2024-04-22T23:27:38.387 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi069 2024-04-22T23:27:38.448 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:38 smithi149 ceph-mon[49797]: from='client.? 172.21.15.69:0/1515937176' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-22T23:27:38.644 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:38 smithi069 systemd[1]: Stopping Ceph mon.smithi069 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:27:39.144 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:38 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069[63129]: 2024-04-22T23:27:38.730+0000 7f4ea30d2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi069 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:27:39.144 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:38 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069[63129]: 2024-04-22T23:27:38.730+0000 7f4ea30d2700 -1 mon.smithi069@0(leader) e3 *** Got Signal Terminated *** 2024-04-22T23:27:39.144 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:38 smithi069 podman[110027]: 2024-04-22 23:27:38.976430751 +0000 UTC m=+0.378307494 container died e4e20d532cda9bf38ecd44f908361cd0c377b3d5aeff38f7e11478e11a0b1c7b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T23:27:39.616 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:39 smithi069 podman[110027]: 2024-04-22 23:27:39.332233085 +0000 UTC m=+0.734109828 container cleanup e4e20d532cda9bf38ecd44f908361cd0c377b3d5aeff38f7e11478e11a0b1c7b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T23:27:39.616 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 22 23:27:39 smithi069 bash[110027]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi069 2024-04-22T23:27:39.755 DEBUG:teuthology.orchestra.run.smithi069:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi069.service' 2024-04-22T23:27:39.799 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T23:27:39.799 INFO:tasks.cephadm.mon.smithi069:Stopped mon.smithi069 2024-04-22T23:27:39.799 INFO:tasks.cephadm.mon.smithi149:Stopping mon.smithi149... 2024-04-22T23:27:39.799 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi149 2024-04-22T23:27:40.113 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:39 smithi149 systemd[1]: Stopping Ceph mon.smithi149 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:27:40.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:40 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149[49793]: 2024-04-22T23:27:40.113+0000 7f1f3386d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi149 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:27:40.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:40 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149[49793]: 2024-04-22T23:27:40.113+0000 7f1f3386d700 -1 mon.smithi149@1(peon) e3 *** Got Signal Terminated *** 2024-04-22T23:27:40.449 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:40 smithi149 podman[75771]: 2024-04-22 23:27:40.226161131 +0000 UTC m=+0.233201246 container died c208daf7eb1c1761a86b585eb5ba2c0e043792e87c3fe22f37333d1fa93fc692 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-22T23:27:40.949 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:40 smithi149 podman[75771]: 2024-04-22 23:27:40.722382605 +0000 UTC m=+0.729422727 container cleanup c208daf7eb1c1761a86b585eb5ba2c0e043792e87c3fe22f37333d1fa93fc692 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-22T23:27:40.949 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:40 smithi149 bash[75771]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149 2024-04-22T23:27:41.279 INFO:journalctl@ceph.mon.smithi149.smithi149.stdout:Apr 22 23:27:40 smithi149 podman[75786]: 2024-04-22 23:27:40.990835527 +0000 UTC m=+0.752787321 container remove c208daf7eb1c1761a86b585eb5ba2c0e043792e87c3fe22f37333d1fa93fc692 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-mon-smithi149, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T23:27:41.308 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@mon.smithi149.service' 2024-04-22T23:27:41.350 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T23:27:41.351 INFO:tasks.cephadm.mon.smithi149:Stopped mon.smithi149 2024-04-22T23:27:41.351 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-22T23:27:41.351 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.0 2024-04-22T23:27:41.644 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:41 smithi069 systemd[1]: Stopping Ceph osd.0 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:27:42.144 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:41 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0[69136]: 2024-04-22T23:27:41.661+0000 7f6852023700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:27:42.144 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:41 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0[69136]: 2024-04-22T23:27:41.661+0000 7f6852023700 -1 osd.0 160 *** Got signal Terminated *** 2024-04-22T23:27:42.144 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:41 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0[69136]: 2024-04-22T23:27:41.661+0000 7f6852023700 -1 osd.0 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:27:47.394 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:47 smithi069 podman[110192]: 2024-04-22 23:27:47.108076675 +0000 UTC m=+5.559561144 container died 881ec8fb453d76bb90c613fab3fde5742aba7a500852923348d77b8fb667c20b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0) 2024-04-22T23:27:47.778 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:47 smithi069 podman[110192]: 2024-04-22 23:27:47.486265596 +0000 UTC m=+5.937750074 container cleanup 881ec8fb453d76bb90c613fab3fde5742aba7a500852923348d77b8fb667c20b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:27:47.778 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:47 smithi069 bash[110192]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0 2024-04-22T23:27:48.114 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:47 smithi069 podman[110209]: 2024-04-22 23:27:47.779238895 +0000 UTC m=+0.673858057 container remove 881ec8fb453d76bb90c613fab3fde5742aba7a500852923348d77b8fb667c20b (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, ceph=True, GIT_CLEAN=True) 2024-04-22T23:27:48.391 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:48 smithi069 podman[110301]: 2024-04-22 23:27:48.018383297 +0000 UTC m=+0.018361191 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:27:48.391 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:48 smithi069 podman[110301]: 2024-04-22 23:27:48.143369515 +0000 UTC m=+0.143347408 container create 6e284ae28c5430e8d0d14732e5f311e139e6cbc775d2c0d5104951e20398faec (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD) 2024-04-22T23:27:48.391 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:48 smithi069 podman[110301]: 2024-04-22 23:27:48.391923198 +0000 UTC m=+0.391901097 container init 6e284ae28c5430e8d0d14732e5f311e139e6cbc775d2c0d5104951e20398faec (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T23:27:48.643 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:48 smithi069 podman[110301]: 2024-04-22 23:27:48.395346393 +0000 UTC m=+0.395324288 container start 6e284ae28c5430e8d0d14732e5f311e139e6cbc775d2c0d5104951e20398faec (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T23:27:48.644 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:48 smithi069 podman[110301]: 2024-04-22 23:27:48.482944338 +0000 UTC m=+0.482922232 container attach 6e284ae28c5430e8d0d14732e5f311e139e6cbc775d2c0d5104951e20398faec (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-22T23:27:48.985 INFO:journalctl@ceph.osd.0.smithi069.stdout:Apr 22 23:27:48 smithi069 podman[110301]: 2024-04-22 23:27:48.699436783 +0000 UTC m=+0.699414677 container died 6e284ae28c5430e8d0d14732e5f311e139e6cbc775d2c0d5104951e20398faec (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T23:27:49.364 DEBUG:teuthology.orchestra.run.smithi069:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.0.service' 2024-04-22T23:27:49.401 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T23:27:49.402 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-22T23:27:49.402 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-22T23:27:49.402 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.1 2024-04-22T23:27:49.644 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:49 smithi069 systemd[1]: Stopping Ceph osd.1 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:27:50.143 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:49 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[71832]: 2024-04-22T23:27:49.757+0000 7ffa39703700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:27:50.143 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:49 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[71832]: 2024-04-22T23:27:49.757+0000 7ffa39703700 -1 osd.1 160 *** Got signal Terminated *** 2024-04-22T23:27:50.144 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:49 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1[71832]: 2024-04-22T23:27:49.757+0000 7ffa39703700 -1 osd.1 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:27:55.507 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:55 smithi069 podman[110409]: 2024-04-22 23:27:55.227278165 +0000 UTC m=+5.606475308 container died 93d880f30ae408c6286afa29cc8d99d13e2da37859f18af9e13f3d2285da4a99 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5) 2024-04-22T23:27:55.893 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:55 smithi069 podman[110409]: 2024-04-22 23:27:55.689982943 +0000 UTC m=+6.069180090 container cleanup 93d880f30ae408c6286afa29cc8d99d13e2da37859f18af9e13f3d2285da4a99 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-22T23:27:55.893 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:55 smithi069 bash[110409]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1 2024-04-22T23:27:56.301 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:55 smithi069 podman[110430]: 2024-04-22 23:27:55.94967471 +0000 UTC m=+0.718613349 container remove 93d880f30ae408c6286afa29cc8d99d13e2da37859f18af9e13f3d2285da4a99 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:27:56.619 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:56 smithi069 podman[110523]: 2024-04-22 23:27:56.206771944 +0000 UTC m=+0.018309568 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:27:56.619 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:56 smithi069 podman[110523]: 2024-04-22 23:27:56.335580919 +0000 UTC m=+0.147118583 container create 9da559fad371b519582fecef224febf5d8fe5469733ed1c6b55a5c3a75520bdd (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-22T23:27:56.619 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:56 smithi069 podman[110523]: 2024-04-22 23:27:56.544909181 +0000 UTC m=+0.356446803 container init 9da559fad371b519582fecef224febf5d8fe5469733ed1c6b55a5c3a75520bdd (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, io.buildah.version=1.33.5) 2024-04-22T23:27:56.619 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:56 smithi069 podman[110523]: 2024-04-22 23:27:56.547992425 +0000 UTC m=+0.359530047 container start 9da559fad371b519582fecef224febf5d8fe5469733ed1c6b55a5c3a75520bdd (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-22T23:27:56.894 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:56 smithi069 podman[110523]: 2024-04-22 23:27:56.619729457 +0000 UTC m=+0.431267095 container attach 9da559fad371b519582fecef224febf5d8fe5469733ed1c6b55a5c3a75520bdd (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-22T23:27:56.894 INFO:journalctl@ceph.osd.1.smithi069.stdout:Apr 22 23:27:56 smithi069 podman[110523]: 2024-04-22 23:27:56.842271758 +0000 UTC m=+0.653809390 container died 9da559fad371b519582fecef224febf5d8fe5469733ed1c6b55a5c3a75520bdd (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416) 2024-04-22T23:27:57.433 DEBUG:teuthology.orchestra.run.smithi069:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.1.service' 2024-04-22T23:27:57.470 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T23:27:57.470 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-22T23:27:57.471 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-22T23:27:57.471 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.2 2024-04-22T23:27:57.817 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:27:57 smithi069 systemd[1]: Stopping Ceph osd.2 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:27:58.143 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:27:57 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2[74469]: 2024-04-22T23:27:57.817+0000 7f1b98a36700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:27:58.143 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:27:57 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2[74469]: 2024-04-22T23:27:57.817+0000 7f1b98a36700 -1 osd.2 160 *** Got signal Terminated *** 2024-04-22T23:27:58.143 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:27:57 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2[74469]: 2024-04-22T23:27:57.817+0000 7f1b98a36700 -1 osd.2 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:28:03.644 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:28:03 smithi069 podman[110629]: 2024-04-22 23:28:03.306309363 +0000 UTC m=+5.608743673 container died 8d3d66dfd7f957e24e15f7d538f1aa7aa063c4c1448e3f6a9037732ac10e94c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416) 2024-04-22T23:28:04.031 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:28:03 smithi069 podman[110629]: 2024-04-22 23:28:03.704762544 +0000 UTC m=+6.007196854 container cleanup 8d3d66dfd7f957e24e15f7d538f1aa7aa063c4c1448e3f6a9037732ac10e94c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-22T23:28:04.031 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:28:03 smithi069 bash[110629]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2 2024-04-22T23:28:04.365 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:28:04 smithi069 podman[110680]: 2024-04-22 23:28:04.032523108 +0000 UTC m=+0.726471402 container remove 8d3d66dfd7f957e24e15f7d538f1aa7aa063c4c1448e3f6a9037732ac10e94c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5) 2024-04-22T23:28:04.643 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:28:04 smithi069 podman[110772]: 2024-04-22 23:28:04.269992825 +0000 UTC m=+0.018565821 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:28:04.644 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:28:04 smithi069 podman[110772]: 2024-04-22 23:28:04.398580499 +0000 UTC m=+0.147153478 container create e7ed75efb8fc5772b132b47b0f1b7271f537b1e9b1af0054b2eced30df494bb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T23:28:04.644 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:28:04 smithi069 podman[110772]: 2024-04-22 23:28:04.609123114 +0000 UTC m=+0.357696092 container init e7ed75efb8fc5772b132b47b0f1b7271f537b1e9b1af0054b2eced30df494bb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, ceph=True, GIT_CLEAN=True) 2024-04-22T23:28:04.644 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:28:04 smithi069 podman[110772]: 2024-04-22 23:28:04.61221901 +0000 UTC m=+0.360791994 container start e7ed75efb8fc5772b132b47b0f1b7271f537b1e9b1af0054b2eced30df494bb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416) 2024-04-22T23:28:05.143 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:28:04 smithi069 podman[110772]: 2024-04-22 23:28:04.709408527 +0000 UTC m=+0.457981515 container attach e7ed75efb8fc5772b132b47b0f1b7271f537b1e9b1af0054b2eced30df494bb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T23:28:05.144 INFO:journalctl@ceph.osd.2.smithi069.stdout:Apr 22 23:28:04 smithi069 podman[110772]: 2024-04-22 23:28:04.906372106 +0000 UTC m=+0.654945086 container died e7ed75efb8fc5772b132b47b0f1b7271f537b1e9b1af0054b2eced30df494bb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-22T23:28:05.559 DEBUG:teuthology.orchestra.run.smithi069:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.2.service' 2024-04-22T23:28:05.604 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T23:28:05.604 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-22T23:28:05.604 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-22T23:28:05.604 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.3 2024-04-22T23:28:05.893 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:05 smithi069 systemd[1]: Stopping Ceph osd.3 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:28:06.393 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:05 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3[77003]: 2024-04-22T23:28:05.948+0000 7f795b1f7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:28:06.393 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:05 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3[77003]: 2024-04-22T23:28:05.948+0000 7f795b1f7700 -1 osd.3 160 *** Got signal Terminated *** 2024-04-22T23:28:06.394 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:05 smithi069 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3[77003]: 2024-04-22T23:28:05.948+0000 7f795b1f7700 -1 osd.3 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:28:08.198 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:07 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:07.824+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:08.948 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:08 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:08.842+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:10.198 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:09 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:09.855+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:10.201 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:09 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:09.745+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:11.198 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:10 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:10.863+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:11.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:10 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:10.754+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:11.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:10 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:10.996+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:11.757 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:11 smithi069 podman[110877]: 2024-04-22 23:28:11.500523797 +0000 UTC m=+5.669068844 container died e7772e3632e1b859e51662440c2b33fc6b478eac536381721b7b6fb8a9aaeaad (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:28:12.064 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:11 smithi069 podman[110877]: 2024-04-22 23:28:11.831887723 +0000 UTC m=+6.000432777 container cleanup e7772e3632e1b859e51662440c2b33fc6b478eac536381721b7b6fb8a9aaeaad (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-22T23:28:12.065 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:11 smithi069 bash[110877]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3 2024-04-22T23:28:12.198 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:11 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:11.899+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:12.199 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:11 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:11.795+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:12.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:11 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:11.947+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:12.393 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:12 smithi069 podman[110896]: 2024-04-22 23:28:12.06630927 +0000 UTC m=+0.563657580 container remove e7772e3632e1b859e51662440c2b33fc6b478eac536381721b7b6fb8a9aaeaad (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T23:28:12.795 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:12 smithi069 podman[110991]: 2024-04-22 23:28:12.409397645 +0000 UTC m=+0.018812901 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:28:12.795 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:12 smithi069 podman[110991]: 2024-04-22 23:28:12.525498483 +0000 UTC m=+0.134913751 container create 957e683f6e766020f2045a785c6f85d32fa5766e8c8c7959b8fb4512906241b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416) 2024-04-22T23:28:12.795 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:12 smithi069 podman[110991]: 2024-04-22 23:28:12.69661361 +0000 UTC m=+0.306028835 container init 957e683f6e766020f2045a785c6f85d32fa5766e8c8c7959b8fb4512906241b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, io.buildah.version=1.33.5) 2024-04-22T23:28:12.795 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:12 smithi069 podman[110991]: 2024-04-22 23:28:12.699840857 +0000 UTC m=+0.309256080 container start 957e683f6e766020f2045a785c6f85d32fa5766e8c8c7959b8fb4512906241b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:28:12.795 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:12 smithi069 podman[110991]: 2024-04-22 23:28:12.796550342 +0000 UTC m=+0.405965566 container attach 957e683f6e766020f2045a785c6f85d32fa5766e8c8c7959b8fb4512906241b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:28:13.117 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:12 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:12.868+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:13.118 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:12 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:12.799+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:13.118 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:12 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:12.953+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:13.143 INFO:journalctl@ceph.osd.3.smithi069.stdout:Apr 22 23:28:13 smithi069 podman[110991]: 2024-04-22 23:28:13.007422945 +0000 UTC m=+0.616838168 container died 957e683f6e766020f2045a785c6f85d32fa5766e8c8c7959b8fb4512906241b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS) 2024-04-22T23:28:13.448 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:13 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:28:13.118+0000 7f7cea077700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:43.454545+0000 front 2024-04-22T23:27:43.454677+0000 (oldest deadline 2024-04-22T23:28:12.255235+0000) 2024-04-22T23:28:13.645 DEBUG:teuthology.orchestra.run.smithi069:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.3.service' 2024-04-22T23:28:13.681 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T23:28:13.681 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-22T23:28:13.681 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-22T23:28:13.681 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.4 2024-04-22T23:28:14.073 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:13 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:13.850+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:14.073 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:13 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:13.799+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:14.074 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:13 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:13.909+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:14.074 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:13 smithi149 systemd[1]: Stopping Ceph osd.4 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:28:14.074 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:14 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:28:14.072+0000 7f7cf1a8b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:28:14.074 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:14 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:28:14.073+0000 7f7cf1a8b700 -1 osd.4 160 *** Got signal Terminated *** 2024-04-22T23:28:14.074 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:14 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:28:14.073+0000 7f7cf1a8b700 -1 osd.4 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:28:14.448 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:14 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:28:14.151+0000 7f7cea077700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:43.454545+0000 front 2024-04-22T23:27:43.454677+0000 (oldest deadline 2024-04-22T23:28:12.255235+0000) 2024-04-22T23:28:15.105 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:14 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:14.857+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:15.106 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:14 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:14.802+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:15.106 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:14 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:14.939+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:15.448 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:15 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:28:15.105+0000 7f7cea077700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:43.454545+0000 front 2024-04-22T23:27:43.454677+0000 (oldest deadline 2024-04-22T23:28:12.255235+0000) 2024-04-22T23:28:16.098 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:15 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:15.815+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:16.098 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:15 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:15.836+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:16.098 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:15 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:15.904+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:16.448 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:16 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:28:16.098+0000 7f7cea077700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:43.454545+0000 front 2024-04-22T23:27:43.454677+0000 (oldest deadline 2024-04-22T23:28:12.255235+0000) 2024-04-22T23:28:16.448 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:16 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:28:16.098+0000 7f7cea077700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.956358+0000 front 2024-04-22T23:27:53.956458+0000 (oldest deadline 2024-04-22T23:28:15.656154+0000) 2024-04-22T23:28:17.142 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:16 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:16.775+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:17.142 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:16 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:16.775+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.180942+0000 front 2024-04-22T23:27:53.180957+0000 (oldest deadline 2024-04-22T23:28:16.080747+0000) 2024-04-22T23:28:17.142 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:16 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:16.807+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:17.143 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:16 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:16.807+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:17.143 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:16 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:16.933+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:17.143 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:16 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:16.933+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:17.448 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:17 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:28:17.143+0000 7f7cea077700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:43.454545+0000 front 2024-04-22T23:27:43.454677+0000 (oldest deadline 2024-04-22T23:28:12.255235+0000) 2024-04-22T23:28:17.448 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:17 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:28:17.143+0000 7f7cea077700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.956358+0000 front 2024-04-22T23:27:53.956458+0000 (oldest deadline 2024-04-22T23:28:15.656154+0000) 2024-04-22T23:28:18.164 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:17 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:17.971+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:18.164 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:17 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:17.971+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:18.164 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:17 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:17.751+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:18.164 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:17 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:17.751+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.180942+0000 front 2024-04-22T23:27:53.180957+0000 (oldest deadline 2024-04-22T23:28:16.080747+0000) 2024-04-22T23:28:18.165 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:17 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:17.816+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:18.165 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:17 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:17.816+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:18.448 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:28:18.165+0000 7f7cea077700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:43.454545+0000 front 2024-04-22T23:27:43.454677+0000 (oldest deadline 2024-04-22T23:28:12.255235+0000) 2024-04-22T23:28:18.448 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4[54754]: 2024-04-22T23:28:18.165+0000 7f7cea077700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.956358+0000 front 2024-04-22T23:27:53.956458+0000 (oldest deadline 2024-04-22T23:28:15.656154+0000) 2024-04-22T23:28:19.198 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:18.947+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:19.198 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:18.947+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:19.199 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:18.786+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:19.199 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:18.786+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.180942+0000 front 2024-04-22T23:27:53.180957+0000 (oldest deadline 2024-04-22T23:28:16.080747+0000) 2024-04-22T23:28:19.199 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:18.809+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:19.199 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:18 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:18.809+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:19.698 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:19 smithi149 podman[75969]: 2024-04-22 23:28:19.548547264 +0000 UTC m=+5.668456830 container died 036a1a8ba3c977e415b72fd1cb8f50fbcf3e4a5c641c42d45b94757b766c2551 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-22T23:28:20.174 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:19 smithi149 podman[75969]: 2024-04-22 23:28:19.957628985 +0000 UTC m=+6.077538559 container cleanup 036a1a8ba3c977e415b72fd1cb8f50fbcf3e4a5c641c42d45b94757b766c2551 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.build-date=20240416) 2024-04-22T23:28:20.174 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:19 smithi149 bash[75969]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4 2024-04-22T23:28:20.174 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:19 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:19.749+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:20.174 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:19 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:19.749+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.180942+0000 front 2024-04-22T23:27:53.180957+0000 (oldest deadline 2024-04-22T23:28:16.080747+0000) 2024-04-22T23:28:20.174 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:19 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:19.857+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:20.175 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:19 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:19.857+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:20.175 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:19 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:19.954+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:20.175 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:19 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:19.954+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:20.448 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:20 smithi149 podman[75988]: 2024-04-22 23:28:20.175625691 +0000 UTC m=+0.623595612 container remove 036a1a8ba3c977e415b72fd1cb8f50fbcf3e4a5c641c42d45b94757b766c2551 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240416) 2024-04-22T23:28:20.802 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:20 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:20.759+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:20.802 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:20 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:20.759+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.180942+0000 front 2024-04-22T23:27:53.180957+0000 (oldest deadline 2024-04-22T23:28:16.080747+0000) 2024-04-22T23:28:20.803 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:20 smithi149 podman[76078]: 2024-04-22 23:28:20.414574101 +0000 UTC m=+0.018285874 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:28:20.803 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:20 smithi149 podman[76078]: 2024-04-22 23:28:20.518516506 +0000 UTC m=+0.122228297 container create 7b2f67a3e23481fe8228eafa9050fbf6da223cbad004edba5bb7208940249b9e (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=) 2024-04-22T23:28:20.803 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:20 smithi149 podman[76078]: 2024-04-22 23:28:20.721124467 +0000 UTC m=+0.324836221 container init 7b2f67a3e23481fe8228eafa9050fbf6da223cbad004edba5bb7208940249b9e (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240416, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-22T23:28:20.803 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:20 smithi149 podman[76078]: 2024-04-22 23:28:20.724515336 +0000 UTC m=+0.328227095 container start 7b2f67a3e23481fe8228eafa9050fbf6da223cbad004edba5bb7208940249b9e (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5) 2024-04-22T23:28:20.803 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:20 smithi149 podman[76078]: 2024-04-22 23:28:20.80436806 +0000 UTC m=+0.408079822 container attach 7b2f67a3e23481fe8228eafa9050fbf6da223cbad004edba5bb7208940249b9e (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-22T23:28:21.198 INFO:journalctl@ceph.osd.4.smithi149.stdout:Apr 22 23:28:21 smithi149 podman[76078]: 2024-04-22 23:28:21.019974939 +0000 UTC m=+0.623686695 container died 7b2f67a3e23481fe8228eafa9050fbf6da223cbad004edba5bb7208940249b9e (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-22T23:28:21.199 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:20 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:20.880+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:21.199 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:20 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:20.880+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:21.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:20 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:20.970+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:21.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:20 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:20.970+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:21.743 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.4.service' 2024-04-22T23:28:21.780 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T23:28:21.781 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-22T23:28:21.781 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-22T23:28:21.781 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.5 2024-04-22T23:28:21.998 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:21 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:21.920+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:21.998 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:21 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:21.920+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:21.998 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:21 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:21.741+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:21.999 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:21 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:21.741+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.180942+0000 front 2024-04-22T23:27:53.180957+0000 (oldest deadline 2024-04-22T23:28:16.080747+0000) 2024-04-22T23:28:21.999 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:21 smithi149 systemd[1]: Stopping Ceph osd.5 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:28:22.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:22 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:21.999+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:22.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:22 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:21.999+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:22.448 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:22 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:22.127+0000 7f61d488e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:28:22.448 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:22 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:22.128+0000 7f61d488e700 -1 osd.5 160 *** Got signal Terminated *** 2024-04-22T23:28:22.448 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:22 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:22.128+0000 7f61d488e700 -1 osd.5 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:28:23.023 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:22 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:22.759+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:23.023 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:22 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:22.759+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.180942+0000 front 2024-04-22T23:27:53.180957+0000 (oldest deadline 2024-04-22T23:28:16.080747+0000) 2024-04-22T23:28:23.024 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:22 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:22.880+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:23.024 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:22 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:22.880+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:23.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:23 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:23.024+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:23.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:23 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:23.024+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:24.054 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:23 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:23.722+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:24.054 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:23 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:23.722+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.180942+0000 front 2024-04-22T23:27:53.180957+0000 (oldest deadline 2024-04-22T23:28:16.080747+0000) 2024-04-22T23:28:24.054 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:23 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:23.880+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:24.054 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:23 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:23.880+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:24.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:24 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:24.055+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:24.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:24 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:24.055+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:25.076 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:24 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:24.731+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:25.077 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:24 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:24.731+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.180942+0000 front 2024-04-22T23:27:53.180957+0000 (oldest deadline 2024-04-22T23:28:16.080747+0000) 2024-04-22T23:28:25.077 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:24 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:24.847+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:25.077 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:24 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:24.847+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:25.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:25 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:25.077+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:25.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:25 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:25.077+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:25.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:25 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:25.077+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:25.948 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:25 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:25.686+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:25.948 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:25 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:25.686+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.180942+0000 front 2024-04-22T23:27:53.180957+0000 (oldest deadline 2024-04-22T23:28:16.080747+0000) 2024-04-22T23:28:25.948 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:25 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:25.686+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:27:59.482104+0000 front 2024-04-22T23:27:59.482062+0000 (oldest deadline 2024-04-22T23:28:25.381962+0000) 2024-04-22T23:28:25.949 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:25 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:25.859+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:25.949 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:25 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:25.859+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:25.949 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:25 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:25.859+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:00.812054+0000 front 2024-04-22T23:28:00.812127+0000 (oldest deadline 2024-04-22T23:28:24.911970+0000) 2024-04-22T23:28:26.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:26 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:26.102+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:26.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:26 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:26.102+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:26.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:26 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:26.102+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:26.948 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:26 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:26.671+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.479650+0000 front 2024-04-22T23:27:44.479527+0000 (oldest deadline 2024-04-22T23:28:07.379489+0000) 2024-04-22T23:28:26.948 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:26 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:26.671+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:53.180942+0000 front 2024-04-22T23:27:53.180957+0000 (oldest deadline 2024-04-22T23:28:16.080747+0000) 2024-04-22T23:28:26.948 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:26 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5[57021]: 2024-04-22T23:28:26.671+0000 7f61d0e82700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:27:59.482104+0000 front 2024-04-22T23:27:59.482062+0000 (oldest deadline 2024-04-22T23:28:25.381962+0000) 2024-04-22T23:28:26.948 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:26 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:26.864+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:26.948 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:26 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:26.864+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:26.949 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:26 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:26.864+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:00.812054+0000 front 2024-04-22T23:28:00.812127+0000 (oldest deadline 2024-04-22T23:28:24.911970+0000) 2024-04-22T23:28:27.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:27 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:27.116+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:27.449 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:27 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:27.116+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:27.449 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:27 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:27.116+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:27.860 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:27 smithi149 podman[76227]: 2024-04-22 23:28:27.554112414 +0000 UTC m=+5.522921235 container died 539c90bd7194849249e61c0c7e3929a7c7e62944c59e34b8d3201eda9b4817eb (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-22T23:28:28.198 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:28 smithi149 podman[76227]: 2024-04-22 23:28:28.061610151 +0000 UTC m=+6.030419000 container cleanup 539c90bd7194849249e61c0c7e3929a7c7e62944c59e34b8d3201eda9b4817eb (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-22T23:28:28.199 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:28 smithi149 bash[76227]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5 2024-04-22T23:28:28.199 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:27 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:27.889+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:28.199 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:27 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:27.889+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:28.199 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:27 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:27.889+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:00.812054+0000 front 2024-04-22T23:28:00.812127+0000 (oldest deadline 2024-04-22T23:28:24.911970+0000) 2024-04-22T23:28:28.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:28 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:28.079+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:28.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:28 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:28.080+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:28.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:28 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:28.080+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:28.698 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:28 smithi149 podman[76243]: 2024-04-22 23:28:28.296459885 +0000 UTC m=+0.737484234 container remove 539c90bd7194849249e61c0c7e3929a7c7e62944c59e34b8d3201eda9b4817eb (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2) 2024-04-22T23:28:29.048 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:28 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:28.906+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:29.048 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:28 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:28.906+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:29.048 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:28 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:28.906+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:00.812054+0000 front 2024-04-22T23:28:00.812127+0000 (oldest deadline 2024-04-22T23:28:24.911970+0000) 2024-04-22T23:28:29.048 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:28 smithi149 podman[76340]: 2024-04-22 23:28:28.640684571 +0000 UTC m=+0.018440781 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:28:29.048 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:28 smithi149 podman[76340]: 2024-04-22 23:28:28.747095074 +0000 UTC m=+0.124851274 container create c2ecd65ae70f983b168465462e5c7840c1b29b93e2988c2f0a4a6a12147656c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-22T23:28:29.048 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:28 smithi149 podman[76340]: 2024-04-22 23:28:28.949551012 +0000 UTC m=+0.327307212 container init c2ecd65ae70f983b168465462e5c7840c1b29b93e2988c2f0a4a6a12147656c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T23:28:29.048 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:28 smithi149 podman[76340]: 2024-04-22 23:28:28.952705597 +0000 UTC m=+0.330461806 container start c2ecd65ae70f983b168465462e5c7840c1b29b93e2988c2f0a4a6a12147656c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-22T23:28:29.048 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:29 smithi149 podman[76340]: 2024-04-22 23:28:29.049480363 +0000 UTC m=+0.427236572 container attach c2ecd65ae70f983b168465462e5c7840c1b29b93e2988c2f0a4a6a12147656c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:28:29.448 INFO:journalctl@ceph.osd.5.smithi149.stdout:Apr 22 23:28:29 smithi149 podman[76340]: 2024-04-22 23:28:29.245379761 +0000 UTC m=+0.623135962 container died c2ecd65ae70f983b168465462e5c7840c1b29b93e2988c2f0a4a6a12147656c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-22T23:28:29.449 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:29 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:29.120+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:29.449 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:29 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:29.120+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:29.449 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:29 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:29.120+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:29.869 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.5.service' 2024-04-22T23:28:29.908 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T23:28:29.908 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-22T23:28:29.908 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-22T23:28:29.908 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.6 2024-04-22T23:28:30.106 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:29 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:29.922+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:30.106 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:29 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:29.922+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:30.106 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:29 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:29.922+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:00.812054+0000 front 2024-04-22T23:28:00.812127+0000 (oldest deadline 2024-04-22T23:28:24.911970+0000) 2024-04-22T23:28:30.106 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:29 smithi149 systemd[1]: Stopping Ceph osd.6 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:28:30.448 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:30 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:30.222+0000 7f8098c39700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:28:30.448 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:30 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:30.222+0000 7f8098c39700 -1 osd.6 160 *** Got signal Terminated *** 2024-04-22T23:28:30.448 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:30 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:30.222+0000 7f8098c39700 -1 osd.6 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:28:30.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:30 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:30.107+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:30.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:30 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:30.107+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:30.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:30 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:30.107+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:31.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:30 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:30.883+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:31.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:30 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:30.883+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:31.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:30 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:30.883+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:00.812054+0000 front 2024-04-22T23:28:00.812127+0000 (oldest deadline 2024-04-22T23:28:24.911970+0000) 2024-04-22T23:28:31.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:31 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:31.086+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:31.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:31 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:31.086+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:31.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:31 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:31.086+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:32.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:31 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:31.927+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:32.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:31 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:31.927+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:32.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:31 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:31.927+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:00.812054+0000 front 2024-04-22T23:28:00.812127+0000 (oldest deadline 2024-04-22T23:28:24.911970+0000) 2024-04-22T23:28:32.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:32 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:32.093+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:32.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:32 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:32.093+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:32.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:32 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:32.093+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:33.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:32 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:32.888+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:33.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:32 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:32.888+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:33.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:32 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:32.888+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:00.812054+0000 front 2024-04-22T23:28:00.812127+0000 (oldest deadline 2024-04-22T23:28:24.911970+0000) 2024-04-22T23:28:33.198 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:33 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:33.128+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:33.198 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:33 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:33.128+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:33.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:33 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:33.128+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:33.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:33 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:33.128+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-04-22T23:28:09.381568+0000 front 2024-04-22T23:28:09.381555+0000 (oldest deadline 2024-04-22T23:28:32.281483+0000) 2024-04-22T23:28:34.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:33 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:33.876+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:34.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:33 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:33.876+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:34.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:33 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:33.876+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:00.812054+0000 front 2024-04-22T23:28:00.812127+0000 (oldest deadline 2024-04-22T23:28:24.911970+0000) 2024-04-22T23:28:34.198 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:34.116+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:34.198 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:34.116+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:34.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:34.116+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:34.199 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:34.116+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-04-22T23:28:09.381568+0000 front 2024-04-22T23:28:09.381555+0000 (oldest deadline 2024-04-22T23:28:32.281483+0000) 2024-04-22T23:28:35.124 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:34.838+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.808114+0000 front 2024-04-22T23:27:44.808288+0000 (oldest deadline 2024-04-22T23:28:09.508133+0000) 2024-04-22T23:28:35.124 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:34.838+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.709692+0000 front 2024-04-22T23:27:54.709669+0000 (oldest deadline 2024-04-22T23:28:16.310138+0000) 2024-04-22T23:28:35.124 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:34 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6[59253]: 2024-04-22T23:28:34.838+0000 7f8091225700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:00.812054+0000 front 2024-04-22T23:28:00.812127+0000 (oldest deadline 2024-04-22T23:28:24.911970+0000) 2024-04-22T23:28:35.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:35 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:35.125+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:35.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:35 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:35.125+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:35.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:35 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:35.125+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:35.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:35 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:35.125+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-04-22T23:28:09.381568+0000 front 2024-04-22T23:28:09.381555+0000 (oldest deadline 2024-04-22T23:28:32.281483+0000) 2024-04-22T23:28:35.906 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:35 smithi149 podman[76445]: 2024-04-22 23:28:35.637511623 +0000 UTC m=+5.506033712 container died b821ba1ee8121c30dd793cdbeb66e72eddce4f842042449e848ee40022ae059f (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-22T23:28:36.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:36 smithi149 podman[76445]: 2024-04-22 23:28:36.081014929 +0000 UTC m=+5.949537021 container cleanup b821ba1ee8121c30dd793cdbeb66e72eddce4f842042449e848ee40022ae059f (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:28:36.198 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:36 smithi149 bash[76445]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6 2024-04-22T23:28:36.198 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:36 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:36.079+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:36.198 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:36 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:36.079+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:36.198 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:36 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:36.079+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:36.198 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:36 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:36.079+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-04-22T23:28:09.381568+0000 front 2024-04-22T23:28:09.381555+0000 (oldest deadline 2024-04-22T23:28:32.281483+0000) 2024-04-22T23:28:36.691 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:36 smithi149 podman[76465]: 2024-04-22 23:28:36.290939547 +0000 UTC m=+0.650182722 container remove b821ba1ee8121c30dd793cdbeb66e72eddce4f842042449e848ee40022ae059f (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.vendor=CentOS) 2024-04-22T23:28:36.947 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:36 smithi149 podman[76552]: 2024-04-22 23:28:36.59665313 +0000 UTC m=+0.018242688 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:28:36.948 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:36 smithi149 podman[76552]: 2024-04-22 23:28:36.70848705 +0000 UTC m=+0.130076594 container create d4ffb19f97a5b5c89293b31bb3c3e137f71248e486930342579288e3f916181c (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T23:28:36.948 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:36 smithi149 podman[76552]: 2024-04-22 23:28:36.934524912 +0000 UTC m=+0.356114456 container init d4ffb19f97a5b5c89293b31bb3c3e137f71248e486930342579288e3f916181c (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5) 2024-04-22T23:28:36.948 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:36 smithi149 podman[76552]: 2024-04-22 23:28:36.9376391 +0000 UTC m=+0.359228646 container start d4ffb19f97a5b5c89293b31bb3c3e137f71248e486930342579288e3f916181c (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020) 2024-04-22T23:28:37.448 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:37 smithi149 podman[76552]: 2024-04-22 23:28:37.042993519 +0000 UTC m=+0.464583079 container attach d4ffb19f97a5b5c89293b31bb3c3e137f71248e486930342579288e3f916181c (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS) 2024-04-22T23:28:37.448 INFO:journalctl@ceph.osd.6.smithi149.stdout:Apr 22 23:28:37 smithi149 podman[76552]: 2024-04-22 23:28:37.235465606 +0000 UTC m=+0.657055151 container died d4ffb19f97a5b5c89293b31bb3c3e137f71248e486930342579288e3f916181c (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2) 2024-04-22T23:28:37.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:37 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:37.095+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:37.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:37 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:37.095+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:37.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:37 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:37.095+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:37.449 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:37 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:37.095+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-04-22T23:28:09.381568+0000 front 2024-04-22T23:28:09.381555+0000 (oldest deadline 2024-04-22T23:28:32.281483+0000) 2024-04-22T23:28:37.828 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.6.service' 2024-04-22T23:28:37.865 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T23:28:37.865 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-22T23:28:37.865 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-22T23:28:37.866 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.7 2024-04-22T23:28:38.143 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:37 smithi149 systemd[1]: Stopping Ceph osd.7 for 6928ed7e-00fc-11ef-bc93-c7b262605968... 2024-04-22T23:28:38.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:38 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:38.145+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:38.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:38 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:38.145+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:38.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:38 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:38.145+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:38.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:38 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:38.145+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-04-22T23:28:09.381568+0000 front 2024-04-22T23:28:09.381555+0000 (oldest deadline 2024-04-22T23:28:32.281483+0000) 2024-04-22T23:28:38.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:38 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:38.190+0000 7effd3b3f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T23:28:38.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:38 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:38.190+0000 7effd3b3f700 -1 osd.7 160 *** Got signal Terminated *** 2024-04-22T23:28:38.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:38 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:38.190+0000 7effd3b3f700 -1 osd.7 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T23:28:39.387 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:39.107+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:39.388 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:39.107+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:39.388 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:39.107+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:39.388 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:39 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:39.107+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-04-22T23:28:09.381568+0000 front 2024-04-22T23:28:09.381555+0000 (oldest deadline 2024-04-22T23:28:32.281483+0000) 2024-04-22T23:28:40.449 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:40 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:40.116+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:40.449 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:40 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:40.116+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:40.449 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:40 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:40.116+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:40.449 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:40 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:40.116+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-04-22T23:28:09.381568+0000 front 2024-04-22T23:28:09.381555+0000 (oldest deadline 2024-04-22T23:28:32.281483+0000) 2024-04-22T23:28:41.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:41 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:41.150+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:41.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:41 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:41.150+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:41.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:41 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:41.150+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:41.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:41 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:41.150+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-04-22T23:28:09.381568+0000 front 2024-04-22T23:28:09.381555+0000 (oldest deadline 2024-04-22T23:28:32.281483+0000) 2024-04-22T23:28:42.447 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:42 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:42.153+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:42.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:42 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:42.153+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:42.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:42 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:42.153+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:42.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:42 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:42.153+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-04-22T23:28:09.381568+0000 front 2024-04-22T23:28:09.381555+0000 (oldest deadline 2024-04-22T23:28:32.281483+0000) 2024-04-22T23:28:43.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:43 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:43.173+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-04-22T23:27:44.577126+0000 front 2024-04-22T23:27:44.577203+0000 (oldest deadline 2024-04-22T23:28:10.377372+0000) 2024-04-22T23:28:43.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:43 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:43.173+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-04-22T23:27:54.378820+0000 front 2024-04-22T23:27:54.378854+0000 (oldest deadline 2024-04-22T23:28:16.678665+0000) 2024-04-22T23:28:43.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:43 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:43.173+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-04-22T23:28:02.380705+0000 front 2024-04-22T23:28:02.380627+0000 (oldest deadline 2024-04-22T23:28:24.080571+0000) 2024-04-22T23:28:43.448 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:43 smithi149 ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7[61521]: 2024-04-22T23:28:43.173+0000 7effcc12b700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-04-22T23:28:09.381568+0000 front 2024-04-22T23:28:09.381555+0000 (oldest deadline 2024-04-22T23:28:32.281483+0000) 2024-04-22T23:28:43.947 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:43 smithi149 podman[76658]: 2024-04-22 23:28:43.698589373 +0000 UTC m=+5.611341112 container died fec23dae1ff2a68ed24b37526311cd0d590fa92ce6f94bf531800b2507caec51 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS) 2024-04-22T23:28:44.427 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:44 smithi149 podman[76658]: 2024-04-22 23:28:44.215330543 +0000 UTC m=+6.128082269 container cleanup fec23dae1ff2a68ed24b37526311cd0d590fa92ce6f94bf531800b2507caec51 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_CLEAN=True, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=) 2024-04-22T23:28:44.427 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:44 smithi149 bash[76658]: ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7 2024-04-22T23:28:44.697 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:44 smithi149 podman[76719]: 2024-04-22 23:28:44.428993884 +0000 UTC m=+0.722686209 container remove fec23dae1ff2a68ed24b37526311cd0d590fa92ce6f94bf531800b2507caec51 (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-22T23:28:45.021 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:44 smithi149 podman[76810]: 2024-04-22 23:28:44.754835975 +0000 UTC m=+0.111004042 container create d78c30ff9f43a08ce1c1df05d5066f61e0877c69d7dd48ea077f0a0c60afaa3f (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-22T23:28:45.021 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:44 smithi149 podman[76810]: 2024-04-22 23:28:44.661745423 +0000 UTC m=+0.017913455 image pull a0434699c1d76af4eea1fc95ff0a22ea73d1e04e609464fbf658d6f39bc7f2fc quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c 2024-04-22T23:28:45.022 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:44 smithi149 podman[76810]: 2024-04-22 23:28:44.957699229 +0000 UTC m=+0.313867255 container init d78c30ff9f43a08ce1c1df05d5066f61e0877c69d7dd48ea077f0a0c60afaa3f (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-22T23:28:45.022 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:44 smithi149 podman[76810]: 2024-04-22 23:28:44.960874782 +0000 UTC m=+0.317042808 container start d78c30ff9f43a08ce1c1df05d5066f61e0877c69d7dd48ea077f0a0c60afaa3f (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T23:28:45.022 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:45 smithi149 podman[76810]: 2024-04-22 23:28:45.023606022 +0000 UTC m=+0.379774048 container attach d78c30ff9f43a08ce1c1df05d5066f61e0877c69d7dd48ea077f0a0c60afaa3f (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-deactivate, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=) 2024-04-22T23:28:45.447 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:45 smithi149 podman[76810]: 2024-04-22 23:28:45.251808374 +0000 UTC m=+0.607976393 container died d78c30ff9f43a08ce1c1df05d5066f61e0877c69d7dd48ea077f0a0c60afaa3f (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, GIT_CLEAN=True) 2024-04-22T23:28:45.795 INFO:journalctl@ceph.osd.7.smithi149.stdout:Apr 22 23:28:45 smithi149 podman[76835]: 2024-04-22 23:28:45.785258595 +0000 UTC m=+0.528283986 container remove d78c30ff9f43a08ce1c1df05d5066f61e0877c69d7dd48ea077f0a0c60afaa3f (image=quay.ceph.io/ceph-ci/ceph@sha256:b8bc99cfccdf4e584cbe106d55b5c5dc774312148ba54d6ca5b660445cf4506c, name=ceph-6928ed7e-00fc-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-1618-43be020, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD) 2024-04-22T23:28:45.801 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6928ed7e-00fc-11ef-bc93-c7b262605968@osd.7.service' 2024-04-22T23:28:45.838 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T23:28:45.838 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-22T23:28:45.838 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-22T23:31:45.356 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-22T23:32:06.954 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T23:32:06.986 DEBUG:teuthology.orchestra.run.smithi149:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T23:32:07.012 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-22T23:32:07.014 DEBUG:teuthology.misc:Transferring archived files from smithi069:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi/7669171/remote/smithi069/crash 2024-04-22T23:32:07.015 DEBUG:teuthology.orchestra.run.smithi069:> sudo tar c -f - -C /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/crash -- . 2024-04-22T23:32:07.061 INFO:teuthology.orchestra.run.smithi069.stderr:tar: /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-22T23:32:07.061 INFO:teuthology.orchestra.run.smithi069.stderr:tar: Error is not recoverable: exiting now 2024-04-22T23:32:07.063 DEBUG:teuthology.misc:Transferring archived files from smithi149:/var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi/7669171/remote/smithi149/crash 2024-04-22T23:32:07.064 DEBUG:teuthology.orchestra.run.smithi149:> sudo tar c -f - -C /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/crash -- . 2024-04-22T23:32:07.098 INFO:teuthology.orchestra.run.smithi149.stderr:tar: /var/lib/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-22T23:32:07.099 INFO:teuthology.orchestra.run.smithi149.stderr:tar: Error is not recoverable: exiting now 2024-04-22T23:32:07.099 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-22T23:32:07.100 DEBUG:teuthology.orchestra.run.smithi069:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-22T23:32:07.130 INFO:tasks.cephadm:Compressing logs... 2024-04-22T23:32:07.130 DEBUG:teuthology.orchestra.run.smithi069:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-22T23:32:07.172 DEBUG:teuthology.orchestra.run.smithi149:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-22T23:32:07.195 INFO:teuthology.orchestra.run.smithi069.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-22T23:32:07.195 INFO:teuthology.orchestra.run.smithi069.stderr:: No such file or directory 2024-04-22T23:32:07.195 INFO:teuthology.orchestra.run.smithi069.stderr:gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.log 2024-04-22T23:32:07.195 INFO:teuthology.orchestra.run.smithi069.stderr:gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mon.smithi069.log 2024-04-22T23:32:07.195 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-22T23:32:07.195 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.log: /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mon.smithi069.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-22T23:32:07.197 INFO:teuthology.orchestra.run.smithi069.stderr:gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-22T23:32:07.197 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.2.log: /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-22T23:32:07.197 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-22T23:32:07.197 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mgr.smithi069.lyiiqg.log 2024-04-22T23:32:07.199 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.0.log: 87.2% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-22T23:32:07.200 INFO:teuthology.orchestra.run.smithi069.stderr: 91.6% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-22T23:32:07.207 INFO:teuthology.orchestra.run.smithi149.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-22T23:32:07.207 INFO:teuthology.orchestra.run.smithi149.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-22T23:32:07.207 INFO:teuthology.orchestra.run.smithi149.stderr:gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi149.ickhha.log 2024-04-22T23:32:07.207 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.log 2024-04-22T23:32:07.207 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi149.ickhha.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-22T23:32:07.207 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mon.smithi149.log 2024-04-22T23:32:07.207 INFO:teuthology.orchestra.run.smithi149.stderr: 87.6% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi149.ickhha.log.gz 2024-04-22T23:32:07.207 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-22T23:32:07.207 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mon.smithi149.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-22T23:32:07.207 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mgr.smithi149.rxqkqk.log 2024-04-22T23:32:07.207 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-22T23:32:07.207 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mgr.smithi149.rxqkqk.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-22T23:32:07.208 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-22T23:32:07.208 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-22T23:32:07.213 INFO:teuthology.orchestra.run.smithi069.stderr:gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-22T23:32:07.213 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mgr.smithi069.lyiiqg.log: gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi069.jiyres.log 2024-04-22T23:32:07.214 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.cephadm.log: 85.9% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-22T23:32:07.215 INFO:teuthology.orchestra.run.smithi069.stderr:gzip -5 --verbose -- /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-22T23:32:07.215 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi069.jiyres.log: 88.9% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi069.jiyres.log.gz 2024-04-22T23:32:07.234 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-volume.log: 91.2% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-22T23:32:07.234 INFO:teuthology.orchestra.run.smithi069.stderr: 91.5% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-22T23:32:07.357 INFO:teuthology.orchestra.run.smithi149.stderr:/var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.cephadm.log: /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-volume.log: 85.8% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-22T23:32:07.414 INFO:teuthology.orchestra.run.smithi149.stderr: 87.2% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-22T23:32:07.439 INFO:teuthology.orchestra.run.smithi149.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-22T23:32:07.634 INFO:teuthology.orchestra.run.smithi069.stderr: 88.4% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mgr.smithi069.lyiiqg.log.gz 2024-04-22T23:32:07.756 INFO:teuthology.orchestra.run.smithi149.stderr: 91.7% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-22T23:32:08.332 INFO:teuthology.orchestra.run.smithi149.stderr: 91.2% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-22T23:32:08.396 INFO:teuthology.orchestra.run.smithi149.stderr: 89.8% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mgr.smithi149.rxqkqk.log.gz 2024-04-22T23:32:09.195 INFO:teuthology.orchestra.run.smithi069.stderr: 91.9% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mon.smithi069.log.gz 2024-04-22T23:32:12.010 INFO:teuthology.orchestra.run.smithi149.stderr: 92.7% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-mon.smithi149.log.gz 2024-04-22T23:32:20.216 INFO:teuthology.orchestra.run.smithi069.stderr: 94.5% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-22T23:32:25.443 INFO:teuthology.orchestra.run.smithi069.stderr: 94.1% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-22T23:32:26.364 INFO:teuthology.orchestra.run.smithi069.stderr: 94.2% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-22T23:32:27.138 INFO:teuthology.orchestra.run.smithi069.stderr: 94.1% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-22T23:32:27.140 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-22T23:32:27.140 INFO:teuthology.orchestra.run.smithi069.stderr:real 0m19.956s 2024-04-22T23:32:27.140 INFO:teuthology.orchestra.run.smithi069.stderr:user 1m10.012s 2024-04-22T23:32:27.140 INFO:teuthology.orchestra.run.smithi069.stderr:sys 0m2.739s 2024-04-22T23:32:29.383 INFO:teuthology.orchestra.run.smithi149.stderr: 93.9% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-22T23:32:36.390 INFO:teuthology.orchestra.run.smithi149.stderr: 94.2% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-22T23:32:54.909 INFO:teuthology.orchestra.run.smithi149.stderr: 94.2% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-22T23:33:02.171 INFO:teuthology.orchestra.run.smithi149.stderr: 94.1% -- replaced with /var/log/ceph/6928ed7e-00fc-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-22T23:33:02.172 INFO:teuthology.orchestra.run.smithi149.stderr: 2024-04-22T23:33:02.173 INFO:teuthology.orchestra.run.smithi149.stderr:real 0m54.987s 2024-04-22T23:33:02.173 INFO:teuthology.orchestra.run.smithi149.stderr:user 1m14.152s 2024-04-22T23:33:02.173 INFO:teuthology.orchestra.run.smithi149.stderr:sys 0m3.293s 2024-04-22T23:33:02.173 INFO:tasks.cephadm:Archiving logs... 2024-04-22T23:33:02.173 DEBUG:teuthology.misc:Transferring archived files from smithi069:/var/log/ceph to /home/teuthworker/archive/adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi/7669171/remote/smithi069/log 2024-04-22T23:33:02.175 DEBUG:teuthology.orchestra.run.smithi069:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-22T23:33:12.039 DEBUG:teuthology.misc:Transferring archived files from smithi149:/var/log/ceph to /home/teuthworker/archive/adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi/7669171/remote/smithi149/log 2024-04-22T23:33:12.040 DEBUG:teuthology.orchestra.run.smithi149:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-22T23:33:22.665 INFO:tasks.cephadm:Removing cluster... 2024-04-22T23:33:22.665 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 --force 2024-04-22T23:33:23.036 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 6928ed7e-00fc-11ef-bc93-c7b262605968 --force 2024-04-22T23:33:23.401 INFO:tasks.cephadm:Removing cephadm ... 2024-04-22T23:33:23.401 DEBUG:teuthology.orchestra.run.smithi069:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-22T23:33:23.417 DEBUG:teuthology.orchestra.run.smithi149:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-22T23:33:23.432 INFO:tasks.cephadm:Teardown complete 2024-04-22T23:33:23.432 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-22T23:33:23.442 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-22T23:33:23.442 DEBUG:teuthology.orchestra.run.smithi069:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T23:33:23.459 DEBUG:teuthology.orchestra.run.smithi149:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T23:33:23.470 INFO:teuthology.orchestra.run.smithi069.stderr:bash: line 1: ntpq: command not found 2024-04-22T23:33:23.475 INFO:teuthology.orchestra.run.smithi069.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T23:33:23.475 INFO:teuthology.orchestra.run.smithi069.stdout:=============================================================================== 2024-04-22T23:33:23.475 INFO:teuthology.orchestra.run.smithi069.stdout:^- hv01.front.sepia.ceph.com 2 6 377 29 +395us[ +395us] +/- 593ms 2024-04-22T23:33:23.475 INFO:teuthology.orchestra.run.smithi069.stdout:^* hv02.front.sepia.ceph.com 3 6 377 30 -55us[ -24us] +/- 57ms 2024-04-22T23:33:23.475 INFO:teuthology.orchestra.run.smithi069.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 28 +120us[ +120us] +/- 67ms 2024-04-22T23:33:23.475 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T23:33:23.486 INFO:teuthology.orchestra.run.smithi149.stderr:bash: line 1: ntpq: command not found 2024-04-22T23:33:23.519 INFO:teuthology.orchestra.run.smithi149.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T23:33:23.519 INFO:teuthology.orchestra.run.smithi149.stdout:=============================================================================== 2024-04-22T23:33:23.519 INFO:teuthology.orchestra.run.smithi149.stdout:^- hv01.front.sepia.ceph.com 2 6 377 29 +371us[ +371us] +/- 593ms 2024-04-22T23:33:23.519 INFO:teuthology.orchestra.run.smithi149.stdout:^* hv02.front.sepia.ceph.com 3 6 377 30 -70us[ -37us] +/- 57ms 2024-04-22T23:33:23.519 INFO:teuthology.orchestra.run.smithi149.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 27 +105us[ +105us] +/- 67ms 2024-04-22T23:33:23.520 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T23:33:23.520 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-22T23:33:23.532 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-22T23:33:23.533 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-22T23:33:23.543 DEBUG:teuthology.orchestra.run.smithi069:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-22T23:33:23.618 DEBUG:teuthology.orchestra.run.smithi149:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-22T23:33:23.745 DEBUG:teuthology.orchestra.run.smithi069:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-22T23:33:23.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T23:33:23.772 DEBUG:teuthology.orchestra.run.smithi149:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-22T23:33:23.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T23:33:23.799 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-22T23:33:23.809 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-22T23:33:23.821 INFO:teuthology.task.internal:Duration was 2063.071709 seconds 2024-04-22T23:33:23.821 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-22T23:33:23.831 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-22T23:33:23.831 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-22T23:33:23.834 DEBUG:teuthology.orchestra.run.smithi149:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-22T23:33:23.867 INFO:teuthology.orchestra.run.smithi069.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T23:33:23.909 INFO:teuthology.orchestra.run.smithi149.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T23:33:24.248 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-22T23:33:24.248 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi069.front.sepia.ceph.com 2024-04-22T23:33:24.248 DEBUG:teuthology.orchestra.run.smithi069:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-22T23:33:24.267 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi149.front.sepia.ceph.com 2024-04-22T23:33:24.268 DEBUG:teuthology.orchestra.run.smithi149:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-22T23:33:24.293 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-22T23:33:24.294 DEBUG:teuthology.orchestra.run.smithi069:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-22T23:33:24.310 DEBUG:teuthology.orchestra.run.smithi149:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-22T23:33:24.359 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-22T23:33:24.359 DEBUG:teuthology.orchestra.run.smithi069:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-22T23:33:24.376 DEBUG:teuthology.orchestra.run.smithi149:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-22T23:33:37.978 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-22T23:33:37.990 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-22T23:33:37.990 DEBUG:teuthology.orchestra.run.smithi069:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-22T23:33:38.014 DEBUG:teuthology.orchestra.run.smithi149:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-22T23:33:38.043 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-22T23:33:38.053 DEBUG:teuthology.orchestra.run.smithi069:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-22T23:33:38.056 DEBUG:teuthology.orchestra.run.smithi149:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-22T23:33:38.075 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern = core 2024-04-22T23:33:38.388 INFO:teuthology.orchestra.run.smithi149.stdout:kernel.core_pattern = core 2024-04-22T23:33:38.418 DEBUG:teuthology.orchestra.run.smithi069:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-22T23:33:38.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T23:33:38.433 DEBUG:teuthology.orchestra.run.smithi149:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-22T23:33:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T23:33:38.449 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-22T23:33:38.460 INFO:teuthology.task.internal:Transferring archived files... 2024-04-22T23:33:38.460 DEBUG:teuthology.misc:Transferring archived files from smithi069:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi/7669171/remote/smithi069 2024-04-22T23:33:38.461 DEBUG:teuthology.orchestra.run.smithi069:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-22T23:33:38.539 DEBUG:teuthology.misc:Transferring archived files from smithi149:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-22_22:45:33-orch:cephadm-wip-adk-testing-2024-04-22-1618-distro-default-smithi/7669171/remote/smithi149 2024-04-22T23:33:38.540 DEBUG:teuthology.orchestra.run.smithi149:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-22T23:33:38.582 INFO:teuthology.task.internal:Removing archive directory... 2024-04-22T23:33:38.582 DEBUG:teuthology.orchestra.run.smithi069:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-22T23:33:38.585 DEBUG:teuthology.orchestra.run.smithi149:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-22T23:33:38.632 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-22T23:33:38.643 INFO:teuthology.task.internal:Not uploading archives. 2024-04-22T23:33:38.643 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-22T23:33:38.671 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-22T23:33:38.672 DEBUG:teuthology.orchestra.run.smithi069:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-22T23:33:38.674 DEBUG:teuthology.orchestra.run.smithi149:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-22T23:33:38.686 INFO:teuthology.orchestra.run.smithi069.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 22 23:33 /home/ubuntu/cephtest 2024-04-22T23:33:38.690 INFO:teuthology.orchestra.run.smithi149.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 22 23:33 /home/ubuntu/cephtest 2024-04-22T23:33:38.691 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-22T23:33:38.703 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-22T23:33:38.728 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2063.071708679199 owner: scheduled_adking@teuthology success: true 2024-04-22T23:33:38.728 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-22T23:33:38.815 INFO:teuthology.run:pass